Ubuntu16.04安装Filebeat

Filebeat官方文档地址

https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-installation.html

下载和安装

curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.3.1-linux-x86_64.tar.gz
tar xzvf filebeat-7.3.1-linux-x86_64.tar.gz

编写filebeat.yml

启动

chmod go-w /home/lintong/software/apache/filebeat-7.3.1-linux-x86_64/filebeat.yml
./filebeat -e -c filebeat.yml

codec.format

codec.format:
    string: '%{[@timestamp]} %{[message]}'

输出

2019-09-13T17:06:51.797Z 123123123123

codec.json

codec.json:
    pretty: true
    escape_html: false

输出

{
  "@timestamp": "2019-09-13T09:08:49.590Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.3.1",
    "topic": "thrift_json_source"
  },
  "host": {
    "name": "lintong-B250M-DS3H"
  },
  "agent": {
    "version": "7.3.1",
    "type": "filebeat",
    "ephemeral_id": "60b93a10-dcce-499b-ae81-0755bfc8bf5c",
    "hostname": "lintong-B250M-DS3H",
    "id": "6ebb0912-ffce-4ddd-9cc8-7bf624e62c78"
  },
  "ecs": {
    "version": "1.0.1"
  },
  "message": "123123123123",
  "log": {
    "file": {
      "path": "/home/lintong/下载/test.log"
    },
    "offset": 0
  },
  "input": {
    "type": "log"
  }
}

如果pretty是false将输出

{"@timestamp":"2019-09-13T09:10:50.164Z","@metadata":{"beat":"filebeat","type":"_doc","version":"7.3.1","topic":"thrift_json_source"},"log":{"file":{"path":"/home/lintong/下载/test.log"},"offset":0},"message":"123123123123","input":{"type":"log"},"ecs":{"version":"1.0.1"},"host":{"name":"lintong-B250M-DS3H"},"agent":{"type":"filebeat","ephemeral_id":"b26075f9-37f8-4d95-9341-fccc4504c1b5","hostname":"lintong-B250M-DS3H","id":"6ebb0912-ffce-4ddd-9cc8-7bf624e62c78","version":"7.3.1"}}

如果需要添加字段

  paths:
    - /home/lintong/下载/test.log
  fields:
    add_field: lintong

输出

{
"@timestamp": "2019-09-16T08:16:06.169Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.3.1",
"topic": "thrift_json_source"
},
"host": {
"name": "lintong-B250M-DS3H"
},
"log": {
"offset": 31,
"file": {
"path": "/home/lintong/下载/test.log"
}
},
"message": "33333333",
"input": {
"type": "log"
},
"fields": {
"add_field": "lintong"
},
"agent": {
"type": "filebeat",
"ephemeral_id": "c16102da-421d-4ff3-90ad-1737451a909d",
"hostname": "lintong-B250M-DS3H",
"id": "6ebb0912-ffce-4ddd-9cc8-7bf624e62c78",
"version": "7.3.1"
},
"ecs": {
"version": "1.0.1"
}
}

在codec.format中添加字段

  codec.format:
     string: '%{[@timestamp]} %{[fields.add_field]} %{[message]}'

输出

2019-09-16T16:18:34.048Z lintong 55555555

如果想添加的字段在json的顶层，就是不在fields字段下层

  paths:
    - /home/lintong/下载/test.log
  fields:
    add_field: lintong
  fields_under_root: true

输出

{
  "@timestamp": "2019-09-16T08:22:43.997Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.3.1",
    "topic": "thrift_json_source"
  },
  "agent": {
    "ephemeral_id": "d8e45d90-6434-4e0d-a6fc-74611b87cbd4",
    "hostname": "lintong-B250M-DS3H",
    "id": "6ebb0912-ffce-4ddd-9cc8-7bf624e62c78",
    "version": "7.3.1",
    "type": "filebeat"
  },
  "log": {
    "offset": 58,
    "file": {
      "path": "/home/lintong/下载/test.log"
    }
  },
  "message": "66666666",
  "add_field": "lintong",
  "input": {
    "type": "log"
  },
  "ecs": {
    "version": "1.0.1"
  },
  "host": {
    "name": "lintong-B250M-DS3H"
  }
}

如果要去掉不要的字段

参考：

https://studygolang.com/articles/10935

和

https://www.elastic.co/guide/en/beats/filebeat/current/drop-fields.html

比如

processors:
- drop_fields:
     fields: ["host", "log", "input","ecs","agent"]

输出

{
  "@timestamp": "2019-09-16T08:55:55.934Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.3.1",
    "topic": "thrift_json_source"
  },
  "message": "33333333",
  "add_field": "lintong"
}

其中@metadata和@timestamp不能在filebeat中去掉