openresty(nginx) 配置 stream 转发

nginx从1.9.0开始，新增加了一个stream模块，用来实现四层协议的转发、代理或者负载均衡等。

（1）关于stream域的模块有哪些？

目前官网上列出的第三方模块、简直就是http模块的镜像、比如access模块访问控制ip和ip段，map模块实现映射、 geo模块实现地理位置映射、等等。使用这些模块的时候一定要看是哪个版本才支持的、比如log模块，只有在nginx-1.11.4才支持。

NGINX的stream相关模块有如下（有些模块特定版本才有，才支持，比如，log模块是NGINX的1.11.4版本）：

ngx_stream_core_module

ngx_stream_access_module

ngx_stream_geo_module

ngx_stream_geoip_module

ngx_stream_js_module

ngx_stream_limit_conn_module

ngx_stream_log_module

ngx_stream_map_module

ngx_stream_proxy_module

ngx_stream_realip_module

ngx_stream_return_module

ngx_stream_split_clients_module

ngx_stream_ssl_module

ngx_stream_ssl_preread_module

ngx_stream_upstream_module

ngx_stream_upstream_hc_module

注意：如果使用 nginx 的 stream 功能，在编译时一定要加上 “--with-stream”

这里使用官方提供的方式在线yum安装openesty，默认已经加上stream 功能了

[root@test sbin]# ./nginx -V
nginx version: openresty/1.19.3.2
built by gcc 9.3.1 20200408 (Red Hat 9.3.1-2) (GCC)
built with OpenSSL 1.1.1k  25 Mar 2021 (running with OpenSSL 1.1.1i  8 Dec 2020)
TLS SNI support enabled
configure arguments: --prefix=/usr/local/openresty/nginx --with-cc-opt='-O2 -DNGX_LUA_ABORT_AT_PANIC -I/usr/local/openresty/zlib/include -I/usr/local/openresty/pcre/include -I/usr/local/openresty/openssl111/include' --add-module=../ngx_devel_kit-0.3.1 --add-module=../echo-nginx-module-0.62 --add-module=../xss-nginx-module-0.06 --add-module=../ngx_coolkit-0.2 --add-module=../set-misc-nginx-module-0.32 --add-module=../form-input-nginx-module-0.12 --add-module=../encrypted-session-nginx-module-0.08 --add-module=../srcache-nginx-module-0.32 --add-module=../ngx_lua-0.10.19 --add-module=../ngx_lua_upstream-0.07 --add-module=../headers-more-nginx-module-0.33 --add-module=../array-var-nginx-module-0.05 --add-module=../memc-nginx-module-0.19 --add-module=../redis2-nginx-module-0.15 --add-module=../redis-nginx-module-0.3.7 --add-module=../ngx_stream_lua-0.0.9 --with-ld-opt='-Wl,-rpath,/usr/local/openresty/luajit/lib -L/usr/local/openresty/zlib/lib -L/usr/local/openresty/pcre/lib -L/usr/local/openresty/openssl111/lib -Wl,-rpath,/usr/local/openresty/zlib/lib:/usr/local/openresty/pcre/lib:/usr/local/openresty/openssl111/lib' --with-cc='ccache gcc -fdiagnostics-color=always' --with-pcre-jit --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-http_v2_module --without-mail_pop3_module --without-mail_imap_module --without-mail_smtp_module --with-http_stub_status_module --with-http_realip_module --with-http_addition_module --with-http_auth_request_module --with-http_secure_link_module --with-http_random_index_module --with-http_gzip_static_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-threads --with-compat --with-stream --with-http_ssl_module

wget https://openresty.org/package/centos/openresty.repo
mv openresty.repo /etc/yum.repos.d/
yum check-update
yum -y install openresty

systemctl start openresty.service

stream模块通常写在events模块下面,与http同一级别.

如下的配置，是监听本机的5678端口转发给stream中upstream的rabbitmq

worker_processes  1;
error_log  logs/error.log;
pid        logs/nginx.pid;

events {
    worker_connections  1024;
}

stream{
    upstream rabbitmq{
        server 192.168.20.100:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.101:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.102:5672 max_fails=2 fail_timeout=5s weight=2;
    }
    server{
        listen 5678; # 任意不占用的端口
        proxy_connect_timeout 10s;
        proxy_timeout 300s;
        proxy_pass rabbitmq; # 注意写法，不带http://
    }
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  logs/access.log  main;

    sendfile        on;
    tcp_nopush      on;

    keepalive_timeout  65;

    gzip  on;
    gzip_min_length 1k;
    gzip_comp_level 7;
    gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;
    gzip_vary on;
    gzip_disable "MSIE [1-6]\.";

    upstream mqweb {
        ip_hash;
        server 192.168.20.100:15672 weight=1 max_fails=2 fail_timeout=30s;
        server 192.168.20.101:15672 weight=1 max_fails=2 fail_timeout=30s;
        server 192.168.20.102:15672 weight=1 max_fails=2 fail_timeout=30s;
    }

    server {
        listen       80;
        server_name  localhost;
        charset      utf-8;
        location / {
            proxy_pass http://mqweb;
            proxy_redirect off;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            client_max_body_size 100m;
            client_body_buffer_size 256k;

            proxy_connect_timeout 60;
            proxy_send_timeout 60;
            proxy_read_timeout 60;

            proxy_buffer_size 256k;
            proxy_buffers 8 256k;
            proxy_busy_buffers_size 512k;
            proxy_temp_file_write_size 512k;
        }
        access_log  logs/host.access.log  main;
    }
}

改变负载均衡的方法

a）least-connected ：对于每个请求，nginx plus选择当前连接数最少的server来处理:

stream{
    upstream rabbitmq{
        least_conn;
        server 192.168.20.100:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.101:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.102:5672 max_fails=2 fail_timeout=5s weight=2;
    }
    server{
        listen 5672;
        proxy_connect_timeout 10s;
        proxy_timeout 300s;
        proxy_pass rabbitmq;
    }
}

b）ip_hash ：客户机的IP地址用作散列键，用于确定应该为客户机的请求选择服务器组中的哪个服务器

stream{
    upstream rabbitmq{
        ip_hash;
        server 192.168.20.100:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.101:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.102:5672 max_fails=2 fail_timeout=5s weight=2;
    }
    server{
        listen 5672;
        proxy_connect_timeout 10s;
        proxy_timeout 300s;
        proxy_pass rabbitmq;
    }
}

注：这个least time均衡方法没有

c）普通的hash算法：nginx plus选择这个server是通过user_defined 关键字，就是IP地址：$remote_addr;

stream{
    upstream rabbitmq{
        hash $remote_addr consistent;
        server 192.168.20.100:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.101:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.102:5672 max_fails=2 fail_timeout=5s weight=2 max_conns=3;;
    }
    server{
        listen 5672;
        proxy_connect_timeout 10s;
        proxy_timeout 300s;
        proxy_pass rabbitmq;
    }
}