关于nginx多层uptstream转发获取客户端真实IP的问题

因为公司有个需求需要获取客户端的真实IP，前端是haproxy，后面是nginx，本来这个需求不难完成，但是难就难在是https请求也就是ssl

由于个人水平有限，在网上爬了很多资料，刚开始的ha是通过tcp代理443端口的，但是无法转发7层的X-Forwarded-For到后面的nginx，那么后面的ng肯定拿不到真实IP了

怎么办呢，网上爬资料

第一在HA上做ssl中断，中断后https协议就变成http协议了这样可以转发到后面nginx上，后面nginx不需要用ssl，但是问题来了，弄好页面出了点问题，并且暂时没法解决，只好暂时作罢

这条路行不通怎么办，那就用nginx做负载吧，谁让自己Low呢，nginx总在7层了吧

说干就干，安装nginx，配置下配置文件，启动，走起

主配置文件如下：

 user  www www;
 worker_processes ;

 error_log  /data/logs/nginx/error.log;
 pid        /var/run/nginx.pid;

 #Specifies the value for maximum file descriptors that can be opened by this process.
 worker_rlimit_nofile ;

 events {
     use epoll;
     worker_connections ;
 }

 http
 {
     include       mime.types;
     #include       proxy.conf;
     default_type  application/octet-stream;

     server_names_hash_bucket_size ;
     client_header_buffer_size 32k;
     large_client_header_buffers  32k;
     client_max_body_size    300m;
     #client_max_body_size    32m;

     #limit_req_zone $baidu_spider zone=baidu_spider:10m rate=15r/m;
     sendfile on;
     tcp_nopush     on;

     keepalive_timeout ;

     tcp_nodelay on;
     #ssi on;
     #ssi_silent_errors on;
     #ssi_types text/shtml;

     fastcgi_connect_timeout ;
     fastcgi_send_timeout ;
     fastcgi_read_timeout ;
     fastcgi_buffer_size 128k;
     fastcgi_buffers  128k;
     fastcgi_busy_buffers_size 128k;
     fastcgi_temp_file_write_size 128k;
     proxy_headers_hash_max_size 51200;
46     proxy_headers_hash_bucket_size 6400;
     gzip on;
     gzip_min_length      2k;
     gzip_buffers          16k;
     gzip_http_version     1.0;
     gzip_comp_level    ;
     gzip_types           text/plain application/x-javascript text/css application/xml;
     gzip_vary         on;
         log_format  access     '$remote_addr,$proxy_add_x_forwarded_for,$http_x_forwarded_for,$remote_user,$time_local,$host,$request,$status,$http_referer,$HTTP_X_UP_CALLING_LINE_ID,$request_time,$http_user_agent  $upstream_addr  $upstream_response_time  $upstream_cache_status';
     #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
         #              '$status $body_bytes_sent "$http_referer" '
         #              '"$http_x_forwarded_for" "$http_user_agent"';

     #upstream backend {
     #    server 127.0.0.1: weight=  max_fails=  fail_timeout=30s;
     #}

     include conf.d/*;

 }

然后我们配置子配置文件：

 upstream ihouse443{
   server 10.0.30.37: max_fails= fail_timeout= weight=;
   server 10.0.30.38: max_fails= fail_timeout= weight=;
   server 10.0.30.39: max_fails= fail_timeout= weight=;

 }

 server {
   listen ;
   server_name www.abc.com;
   access_log /data/logs/nginx/abc_access.log access;
   error_log  /data/logs/nginx/abc_error.log ;
   ssl on;
   ssl_certificate /data/ifengsite/htdocs/abc.sss.com.crt;
   ssl_certificate_key /data/ifengsite/htdocs/abc.sss.com.key;

   ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDH:AES:HIGH:!aNULL:!MD5:!ADH:!DH;
   ssl_protocols TLSv1 TLSv1. TLSv1.;

   #location /stub_status {
   #  stub_status;
   #}

   location / {
     proxy_pass https://ihouse443;
     proxy_redirect off;
     proxy_set_header Host $host;
     proxy_set_header X-Real-IP $remote_addr;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     #proxy_set_header X-Forwarded-For $http_x_forwarded_for;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
     proxy_max_temp_file_size ;
     proxy_connect_timeout ;
     proxy_send_timeout ;
     proxy_read_timeout ;
     proxy_buffer_size 4k;
     proxy_buffers  32k;
     proxy_busy_buffers_size 64k;
     proxy_temp_file_write_size 64k;
     #include        fastcgi_params;
   }
 }

 server {
   listen ;
   server_name *.abc.def.com;
   access_log /data/logs/nginx/ihouse_access1.log access;
   error_log  /data/logs/nginx/ihouse_error.log ;
   ssl on;
   ssl_certificate /data/ifengsite/htdocs/_.sss.xxx.com.crt;
   ssl_certificate_key /data/ifengsite/htdocs/_.sss.xxx.com.key;
   ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDH:AES:HIGH:!aNULL:!MD5:!ADH:!DH;
   ssl_protocols TLSv1 TLSv1. TLSv1.;
   location / {
     proxy_pass https://ihouse443;
     proxy_redirect off;
     proxy_set_header Host $host;
     proxy_set_header X-Real-IP $remote_addr;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     #proxy_set_header X-Forwarded-For $http_x_forwarded_for;
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
     proxy_max_temp_file_size ;
     proxy_connect_timeout ;
     proxy_send_timeout ;
     proxy_read_timeout ;
     proxy_buffer_size 4k;
     proxy_buffers  32k;
     proxy_busy_buffers_size 64k;
     proxy_temp_file_write_size 64k;
     #include        fastcgi_params;
   }
 }

然后启动nginx，去后面的ng上一看，哎怎么没真实IP呢？都是内网IP

后来在网上找了点 资料，如下解释，这是摘抄的

下面来分析请求头到达Nginx负载均衡服务器的情况；在默认情况下，Nginx并不会对X-Forwarded-For头做任何的处理，除非用户使用proxy_set_header 参数设置：

1. proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;

$proxy_add_x_forwarded_for变量包含客户端请求头中的"X-Forwarded-For"，与$remote_addr用逗号分开，如果没有"X-Forwarded-For" 请求头，则$proxy_add_x_forwarded_for等于$remote_addr。$remote_addr变量的值是客户端的IP。

当Nginx设置X-Forwarded-For于$proxy_add_x_forwarded_for后会有两种情况发生：

1、如果从CDN过来的请求没有设置X-Forwarded-For头（通常这种事情不会发生），而到了我们这里Nginx设置将其设置为$proxy_add_x_forwarded_for的话，X-Forwarded-For的信息应该为CDN的IP，因为相对于Nginx负载均衡来说客户端即为CDN，这样的话，后端的web程序时死活也获得不了真实用户的IP的。

2、CDN设置了X-Forwarded-For，我们这里又设置了一次，且值为$proxy_add_x_forwarded_for的话，那么X-Forwarded-For的内容变成 ”客户端IP,Nginx负载均衡服务器IP“如果是这种情况的话，那后端的程序通过X-Forwarded-For获得客户端IP，则取逗号分隔的第一项即可。

如上两点所说，如果我们知道了CDN设置了X-Forwarded-For信息，且只有客户端真实的IP的话，那么我们的Nginx负载均衡服务器可以不必理会该头，让它默认即可。

其实Nginx中还有一个$http_x_forwarded_for变量，这个变量中保存的内容就是请求中的X-Forwarded-For信息。如果后端获得X-Forwarded-For信息的程序兼容性不好的话（没有考虑到X-Forwarded-For含有多个IP的情况），最好就不要将X-Forwarded-For设置为 $proxy_add_x_forwarded_for。应该设置为$http_x_forwarded_for或者干脆不设置！

#########################################################################################################

找到了以上内容，然后就测试

首先上面配置文件蓝色内容是后来加上去的，红色内容是原本就有的

蓝色的配置加上去后，红色的注释掉后，后端的nginx就可以获取真实IP了，具体原理就是上面所说的但是我还是不太明白，后续还要好好研究下。

附上后端nginx的配置

 user  www www;
 worker_processes ;

 error_log  /data/logs/nginx/error.log;
 pid        /var/run/nginx.pid;

 #Specifies the value for maximum file descriptors that can be opened by this process.
 worker_rlimit_nofile ;

 events {
     use epoll;
     worker_connections ;
 }

 http
 {
     include       mime.types;
     #include       proxy.conf;
     default_type  application/octet-stream;

     server_names_hash_bucket_size ;
     client_header_buffer_size 32k;
     large_client_header_buffers  32k;
     client_max_body_size    300m;
     #client_max_body_size    32m;

     #limit_req_zone $baidu_spider zone=baidu_spider:10m rate=15r/m;
     sendfile on;
     tcp_nopush     on;

     keepalive_timeout ;

     tcp_nodelay on;
     #ssi on;
     #ssi_silent_errors on;
     #ssi_types text/shtml;

     fastcgi_connect_timeout ;
     fastcgi_send_timeout ;
     fastcgi_read_timeout ;
     fastcgi_buffer_size 128k;
     fastcgi_buffers  128k;
     fastcgi_busy_buffers_size 128k;
     fastcgi_temp_file_write_size 128k;     gzip on;
     gzip_min_length      2k;
     gzip_buffers          16k;
     gzip_http_version     1.0;
     gzip_comp_level    ;
     gzip_types           text/plain application/x-javascript text/css application/xml;
     gzip_vary         on;
         log_format  access     '$remote_addr,$http_x_forwarded_for,$remote_user,$time_local,$host,$request,$status,$http_referer,$HTTP_X_UP_CALLING_LINE_ID,$request_time,$http_user_agent  $upstream_addr  $upstream_response_time  $upstream_cache_status';
     #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
         #              '$status $body_bytes_sent "$http_referer" '
         #              '"$http_x_forwarded_for" "$http_user_agent"';

     #upstream backend {
     #    server 127.0.0.1: weight=  max_fails=  fail_timeout=30s;
     #}

     include conf.d/*;

 }

子配置文件

upstream ihouse443{
  server 10.0.30.38: max_fails= fail_timeout= weight=;
  server 10.0.30.38: max_fails= fail_timeout= weight=;
  server 10.0.10.50: max_fails= fail_timeout= weight=;

}

server {
  listen ;
  server_name www.abc.com;
  access_log /data/logs/nginx/sss_access.log access;
  error_log  /data/logs/nginx/ihouse_error.log ;
  ssl on;
  ssl_certificate /data/ifengsite/htdocs/ss.xx.com.crt;
  ssl_certificate_key /data/ifengsite/htdocs/ssxx.com.key;

  ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDH:AES:HIGH:!aNULL:!MD5:!ADH:!DH;
  ssl_protocols TLSv1 TLSv1. TLSv1.;

  #location /stub_status {
  #  stub_status;
  #}

  location / {
    proxy_pass https://ihouse443;
    proxy_redirect off;
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
    proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
    proxy_max_temp_file_size ;
    proxy_connect_timeout ;
    proxy_send_timeout ;
    proxy_read_timeout ;
    proxy_buffer_size 4k;
    proxy_buffers  32k;
    proxy_busy_buffers_size 64k;
    proxy_temp_file_write_size 64k;
    #include        fastcgi_params;
  }
}

后端nginx不需要配置

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;