kong更改日志格式

基于业务的需求，需要对网关层的日志进行适当定制，以满足使用kibana的制图。

对于kong的日志格式更改，可查看到的资料都过于繁琐，特此记录。

修改kong的日志格式

 #  ctl edit deployment ingress-kong -n kong
 ...
 spec:
      containers:
      - env:
        - name: KONG_PLUGINS
          value: forward-auth-request,prometheus
        - name: KONG_LUA_PACKAGE_PATH
          value: /opt/?.lua;;
        - name: KONG_PROXY_LISTEN
          value: 0.0.0.0:8000, 0.0.0.0:8443 ssl http2
        - name: KONG_ADMIN_LISTEN
          value: 127.0.0.1:8444 ssl
        - name: KONG_STATUS_LISTEN
          value: 0.0.0.0:8100
        - name: KONG_DATABASE
          value: "off"
        - name: KONG_NGINX_WORKER_PROCESSES
          value: "1"
        - name: KONG_ADMIN_ACCESS_LOG
          value: /dev/stdout custom_fmt
        - name: KONG_ADMIN_ERROR_LOG
          value: /dev/stderr
        - name: KONG_PROXY_ACCESS_LOG
          value: /dev/stdout custom_fmt
        - name: KONG_PROXY_ERROR_LOG
          value: /dev/stderr
        - name: KONG_NGINX_HTTP_LOG_FORMAT
          value: custom_fmt '$remote_addr - $remote_user [$time_local] "$request"
            $status "$upstream_status" $body_bytes_sent "$http_referer" "$http_user_agent"
            "$request_time" "$upstream_response_time"'
 ...

查看日志格式

1xx.xxx.xxx.xx - - [26/Jan/2021:08:14:59 +0000] "PUT /api/xxxx/xxs/xxxx38a458 HTTP/2.0" 200 "200" 133 "https://www.baidu.com/xx/exxxor/e11xxxxxxxxxx47c/?lang=zh" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36" "0.072" "0.068"

kong 获取客户端真实ip

# ctl edit svc kong-proxy -n kong
...
spec:
  ...
  externalTrafficPolicy: Local
  ...

logstash 解析kong日志格式

filter {   # kong的单独索引
      if [type] == "kong" {
        grok {
          match => {
            "message" => '(?<clientip>[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}) - - \[(?<requesttime>[^ ]+ \+[0-9]+)\] "(?<requesttype>[A-Z]+) (?<requesturl>[^ ]+) HTTP/\d.\d" (?<status>[0-9]{3}) "(?<upstream_status>[0-9]+)" (?<body_bytes_sent>[0-9]+) "(?<http_referer>[^"]+)" "(?<http_user_agent>[^"]+)" "(?<request_time>[0-9]+\.[0-9]+)" "(?<upstream_response_time>[0-9]+\.[0-9]+)"'
          }
        }
        date {
          match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
          target => "@timestamp"
        }
        geoip {
          source => "clientip"
          target => ["geoip"]
          fi
        }
     }
 }