Kubernetes fluentd+elasticsearch+kibana

前提：dns服务，k8s集群

下载kubernetes，并解压

https://github.com/kubernetes/kubernetes/releases

tar zxvf kubernetes.tar.gz

cd kubernetes/cluster/addons/fluentd-elasticsearch

部署elasticsearch

[root@k8s_master fluentd]# cat Elasticsearch-rc.yaml
apiVersion: v1
kind: ReplicationController
metadata:
  name: elasticsearch-logging-v1
  namespace: kube-system
  labels:
    k8s-app: elasticsearch-logging
    version: v1
    kubernetes.io/cluster-service: "true"
spec:
  replicas:
  selector:
    k8s-app: elasticsearch-logging
    version: v1
  template:
    metadata:
      labels:
        k8s-app: elasticsearch-logging
        version: v1
        kubernetes.io/cluster-service: "true"
    spec:
      containers:
      - image: elasticsearch
        name: elasticsearch-logging
        resources:
          # need more cpu upon initialization, therefore burstable class
          limits:
            cpu: 1000m
          requests:
            cpu: 500m
        ports:
        - containerPort:
          name: db
          protocol: TCP
        - containerPort:
          name: transport
          protocol: TCP
        volumeMounts:
        - name: es-persistent-storage
          mountPath: /data
      volumes:
      - name: es-persistent-storage
        emptyDir: {}

[root@k8s_master fluentd]# cat Elasticsearch-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: elasticsearch-logging
  namespace: kube-system
  labels:
    k8s-app: elasticsearch-logging
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "Elasticsearch"
spec:
  ports:
  - port:
    protocol: TCP
    targetPort: db
  selector:
    k8s-app: elasticsearch-logging

部署kibana

[root@k8s_master fluntd]# cat kibana-rc.yaml
apiVersion: v1
kind: ReplicationController
metadata:
  name: kibana-logging-v1
  namespace: kube-system
  labels:
    k8s-app: kibana-logging
    version: v1
    kubernetes.io/cluster-service: "true"
spec:
  replicas:
  selector:
      k8s-app: kibana-logging
      version: v1
  template:
    metadata:
      labels:
        k8s-app: kibana-logging
        version: v1
        kubernetes.io/cluster-service: "true"
    spec:
      containers:
      - name: kibana-logging
        image: kibana
        resources:
          # keep request = limit to keep this container in guaranteed class
          limits:
            cpu: 900m
          requests:
            cpu: 500m
        env:
          - name: "ELASTICSEARCH_URL"
            value: "http://elasticsearch-logging:9200"
        ports:
        - containerPort:
          name: ui
          protocol: TCP
[root@k8s_master fluntd]# cat kibana-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: kibana-logging
  namespace: kube-system
  labels:
    name: kibana-logging
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "Kibana"
spec:
  ports:
  - port:
    protocol: TCP
    targetPort: ui
    nodePort:
  type: NodePort
  selector:
    k8s-app: kibana-logging

部署fluentd

[root@k8s_master fluntd]# cat fluentd-pod.yaml
apiVersion: v1
kind: Pod
metadata:
  name: fluentd-elasticsearch
  namespace: kube-system
spec:
  containers:
  - name: fluentd-elasticsearch
    image: registry.cn-hangzhou.aliyuncs.com/kb_containers/fluentd-elasticsearch:1.20
    imagePullPolicy: IfNotPresent
    resources:
      limits:
         memory: 200Mi
      requests:
         cpu: 100m
         memory: 200Mi
    volumeMounts:
    - name: varlog
      mountPath: /var/log
    - name: varlibdockercontainers
      mountPath: /var/lib/docker/containers
      readOnly: true
  terminationGracePeriodSeconds:
  volumes:
  - name: varlog
    hostPath:
      path: /var/log
  - name: varlibdockercontainers
    hostPath:
      path: /var/lib/docker/containers

上述全部创建成功以后(如果不成功，使用kubectl describe/logs 命令组合使用查看相关event和logs看结构即可)，检查相关资源

kubectl get pods --namespace=kube-system
kubectl get rc --namespace=kube-system
kubectl get service --namespace=kube-system
kubectl get endpoints --namespace=kube-system

kubectl cluster-info　　　　#查看kibana和elasticsearch的连接信息

查看elasticsearch是否有索引

api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging/_cat/indices?v　　#查看是否有索引信息，如果没有，可能是没有日志信息

本例使用fluentd收集容器相关信息，日志存储路径为/var/log/containers目录下以.log结尾的文件，如果没有，则需要查看docker相关配置文件，如下

[root@k8s_node1 containers]# cat /etc/sysconfig/docker

# /etc/sysconfig/docker

# Modify these options if you want to change the way the docker daemon runs
#OPTIONS='--selinux-enabled --log-driver=journald --signature-verification=false'　　　　#默认是将容器日志写到/var/log/message
OPTIONS='--selinux-enabled --log-driver=json-file --signature-verification=false'　　　　#需要修改为 json-file写到Container目录下
if [ -z "${DOCKER_CERT_PATH}" ]; then
    DOCKER_CERT_PATH=/etc/docker
fi

# Do not add registries in this file anymore. Use /etc/containers/registries.conf
# from the atomic-registries package.
#

# docker-latest daemon can be used by starting the docker-latest unitfile.
# To use docker-latest client, uncomment below lines
#DOCKERBINARY=/usr/bin/docker-latest
#DOCKERDBINARY=/usr/bin/dockerd-latest
#DOCKER_CONTAINERD_BINARY=/usr/bin/docker-containerd-latest
#DOCKER_CONTAINERD_SHIM_BINARY=/usr/bin/docker-containerd-shim-latest

重启docker服务即可

systemctl daemon-reload
systemctl restart docker