filebeat开启自带模块收集日志如何辨别日志来源等

filebeat启动自带模块后，日志先输出到Redis中

比如开启了system模块日志和redis模块日志

在Redis中查看收集过来的日志时，可以看到如下的这些信息

system日志信息

{
  "@timestamp": "2019-09-02T04:10:20.423Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.3.0",
    "pipeline": "filebeat-7.3.0-system-syslog-pipeline"
  },
  "ecs": {
    "version": "1.0.1"
  },
  "host": {
    "os": {
      "name": "CentOS Linux",
      "kernel": "3.10.0-957.21.3.el7.x86_64",
      "codename": "Core",
      "platform": "centos",
      "version": "7 (Core)",
      "family": "redhat"
    },
    "id": "35a7a3c7af8f44188f7095d5291a188e",
    "containerized": false,
    "name": "bogon",
    "hostname": "bogon",
    "architecture": "x86_64"
  },
  "service": {
    "type": "system"
  },
  "input": {
    "type": "log"
  },
  "event": {
    "module": "system",
    "dataset": "system.syslog",
    "timezone": "+08:00"
  },
  "fileset": {
    "name": "syslog"
  },
  "agent": {
    "hostname": "bogon",
    "id": "a44c8bbc-723c-4982-84f8-bad50c80fac9",
    "version": "7.3.0",
    "type": "filebeat",
    "ephemeral_id": "50725221-8fe5-48be-af66-89e43fadf1c2"
  },
  "log": {
    "offset": 21029,
    "file": {
      "path": "/var/log/messages"
    }
  },
  "message": "Sep  2 12:10:10 bogon filebeat: 2019-09-02T12:10:10.357+0800#011INFO#011crawler/crawler.go:139#011Stopping Crawler"
}

redis日志信息

{
  "@timestamp": "2019-09-02T05:33:45.984Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.3.0",
    "pipeline": "filebeat-7.3.0-redis-log-pipeline"
  },
  "service": {
    "type": "redis"
  },
  "host": {
    "hostname": "bogon",
    "architecture": "x86_64",
    "os": {
      "codename": "Core",
      "platform": "centos",
      "version": "7 (Core)",
      "family": "redhat",
      "name": "CentOS Linux",
      "kernel": "3.10.0-957.21.3.el7.x86_64"
    },
    "id": "35a7a3c7af8f44188f7095d5291a188e",
    "containerized": false,
    "name": "bogon"
  },
  "agent": {
    "hostname": "bogon",
    "id": "a44c8bbc-723c-4982-84f8-bad50c80fac9",
    "version": "7.3.0",
    "type": "filebeat",
    "ephemeral_id": "50725221-8fe5-48be-af66-89e43fadf1c2"
  },
  "ecs": {
    "version": "1.0.1"
  },
  "event": {
    "dataset": "redis.log",
    "module": "redis"
  },
  "fileset": {
    "name": "log"
  },
  "input": {
    "type": "log"
  },
  "log": {
    "offset": 21001,
    "file": {
      "path": "/var/log/redis/redis_6379.log"
    }
  },
  "message": "1812:M 02 Sep 2019 13:33:45.068 * Background saving started by pid 2682"
}

根据下图所示，有两处地方可以用来判断来源

可以根据这俩的不同在logstash中判断来源，进而在elasticsearch中生成不同的索引