filebeat获取nginx的access日志配置

filebeat获取nginx的access日志配置

产生nginx日志的服务器即生产者服务器配置：
拿omp.chinasoft.com举例：
.nginx.conf主配置文件添加日志格式

log_format  log_json  '{ "remote_addr": "$remote_addr", '
                          '"remote_user": "$remote_user", '
                          '"time_local": "$time_local", '
                          '"request_url": "$scheme://$host$request_uri", '
                          '"status": "$status", '
                          '"body_bytes_sent": "$body_bytes_sent", '
                          '"http_referer": "$http_referer", '
                          '"http_user_agent": "$http_user_agent", '
                          '"http_x_forwarded_for": "$http_x_forwarded_for", '
                          '"request_time": "$request_time", '
                          '"timestamp": "$msec", '
                          '"request_method": "$request_method", '
                          '"root_id": "$root_id", '
                          '"trace_id": "$http_trace_id", '
                          '"span_id": "$http_span_id" '
                        '}';

.vhosts虚拟主机配置

a.server部分添加追踪参数
####### 添加链路追踪参数 #########
set $app_name "omp.chinasoft.com";
set $root_id $pid.$msec.$remote_addr.$connection.$connection_requests;
if ($http_root_id != "")
{
    set $root_id $http_root_id;
}
.location部分添加
####### 添加链路追踪参数 #########

fastcgi_param ROOT_ID $root_id;
fastcgi_param APP_NAME $app_name;

.调整日志的路径
error_log /data/www/logs/nginx_log/error/omp.chinasoft.com_error.log;
access_log /data/www/logs/nginx_log/access/omp.chinasoft.com_access.log log_json;

# 具体配置
# cat /usr/local/nginx/config/vhost.d/omp.chinasoft.com
server {
    listen ;
    server_name omp.chinasoft.com;
    return  https://$server_name$request_uri;
}

server {
    listen  ssl;
    ssl_certificate /usr/local/nginx/config/cert2016/chinasoft_com.crt;
    ssl_certificate_key /usr/local/nginx/config/cert2016/chinasoft_com.key;
    ssl_session_cache shared:SSL1:20m;
    ssl_session_timeout 60m;
    ssl_protocols TLSv1 TLSv1. TLSv1.;

    server_name omp.chinasoft.com;
    index index.html index.php;
    root /var/www/vhosts/cloud_oa/public;

    client_max_body_size 800M;

    add_header Ws-From 'http/1.1 2.1.1.1 stable';

    set $app_name "omp.chinasoft.com";
    set $root_id $pid.$msec.$remote_addr.$connection.$connection_requests;
    if ($http_root_id != "")
    {
        set $root_id $http_root_id;
    }

    location / {
        try_files $uri $uri/ /index.php?$query_string;
    }

    location ~ /*\.php {
        fastcgi_param ROOT_ID $root_id;
        fastcgi_param APP_NAME $app_name;
        include fastcgi_params;
        fastcgi_param SCRIPT_FILENAME  $document_root$fastcgi_script_name;
        fastcgi_param APP_ENV "production";
        fastcgi_pass unix:/tmp/php5.6-fpm.sock;
        fastcgi_index index.php;
    }

    location ~ .*\.(xml|gif|jpg|jpeg|png|bmp|swf|woff|woff2|ttf|js|css)$ {
        expires 30d;
    }

    #error_log /var/log/nginx/omp.chinasoft.com_error.log;
    #access_log /var/log/nginx/omp.chinasoft.com_access.log;
    error_log /data/www/logs/nginx_log/error/omp.chinasoft.com_error.log;
    access_log /data/www/logs/nginx_log/access/omp.chinasoft.com_access.log log_json;
}

3.filebeat的配置

# cat /usr/local/filebeat-7.3.1-linux-x86_64/modules.d/nginx.yml
# Module: nginx
# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.3/filebeat-module-nginx.html
filebeat.inputs:
# 采集nginx错误日志
- type: log
  enabled: true
  paths:
    - /data/www/logs/nginx_log/error/*_error.log
  fields:
    log_type: ngx_error
  fields_under_root: true
  tail_files: true
  multiline.pattern: '^\['
  multiline.negate: true
  multiline.match: after

# 采集nginx访问日志
- type: log
  enabled: true
  paths:
    - /data/www/logs/nginx_log/access/*_access.log
  fields:
    log_type: ngx_access
  fields_under_root: true
  tail_files: true
  exclude_lines: ['\.(xml|gif|jpg|jpeg|png|bmp|swf|woff|woff2|ttf|js|css|svg|ico)']

# 输出
output.kafka:
  hosts: ["1.1.1.1:19092"]
  topic: filebeats-log

# 启动filebeat命令
./filebeat -c modules.d/nginx.yml
/usr/local/filebeat-7.3.1-linux-x86_64/filebeat -c /usr/local/filebeat-7.3.1-linux-x86_64/modules.d/nginx.yml &

查看kafka日志具体内容的命令：
kafka配置路径，可以查看日志存放的路径：
config/server.config

/usr/local/elk/kafka/bin/kafka-run-class.sh kafka.tools.DumpLogSegments --files /data/kafka/logs/filebeats-log-omp-0/00000000000000000000.log --print-data-log

filebeat udp监听配置
filebeat.inputs:

# 监听udp
- type: udp
  enabled: true
  host: "0.0.0.0:9999"

output.kafka:
  hosts: ["10.10.18.242:9092"]
  topic: filebeats-log

# 采集者
processors:
- drop_fields:
    fields: ["ecs",  "input", "agent", "@metadata", "tags"]