kubernetes监控和性能分析工具：heapster+influxdb+grafana

1.部署heapster

下载 heapster 相关 yaml 文件

[root@master dashboard]# wget https://raw.githubusercontent.com/kubernetes-retired/heapster/master/deploy/kube-config/influxdb/grafana.yaml
[root@master dashboard]# wget https://raw.githubusercontent.com/kubernetes-retired/heapster/master/deploy/kube-config/influxdb/influxdb.yaml
[root@master dashboard]# wget https://raw.githubusercontent.com/kubernetes-retired/heapster/master/deploy/kube-config/influxdb/heapster.yaml
[root@master dashboard]# wget https://raw.githubusercontent.com/kubernetes-retired/heapster/master/deploy/kube-config/rbac/heapster-rbac.yaml
[root@master dashboard]# cat  heapster.yaml | grep image
        image: k8s.gcr.io/heapster-amd64:v1.5.4
        imagePullPolicy: IfNotPresent
[root@master dashboard]#  cat  grafana.yaml | grep image
        image: k8s.gcr.io/heapster-grafana-amd64:v5.0.4
[root@master dashboard]# cat influxdb.yaml | grep image
        image: k8s.gcr.io/heapster-influxdb-amd64:v1.5.2

1.1在node上pull和tag被墙的镜像

[root@master dashboard]# docker pull mirrorgooglecontainers/heapster-amd64:v1.5.4
[root@master dashboard]# docker pull mirrorgooglecontainers/heapster-grafana-amd64:v5.0.4
[root@master dashboard]# docker pull mirrorgooglecontainers/heapster-influxdb-amd64:v1.5.2

[root@master dashboard]# docker tag mirrorgooglecontainers/heapster-amd64:v1.5.4 k8s.gcr.io/heapster-amd64:v1.5.4
[root@master dashboard]# docker tag mirrorgooglecontainers/heapster-grafana-amd64:v5.0.4 k8s.gcr.io/heapster-grafana-amd64:v5.0.4
[root@master dashboard]# docker tag mirrorgooglecontainers/heapster-influxdb-amd64:v1.5.2 k8s.gcr.io/heapster-influxdb-amd64:v1.5.2
[root@master dashboard]# ls heapster/
grafana.yaml  heapster-rbac.yaml  heapster.yaml  influxdb.yaml
[root@master dashboard]# kubectl apply -f ./heapster/
deployment.extensions/monitoring-grafana created
service/monitoring-grafana created
clusterrolebinding.rbac.authorization.k8s.io/heapster created
serviceaccount/heapster created
deployment.extensions/heapster created
service/heapster created
deployment.extensions/monitoring-influxdb created
service/monitoring-influxdb created
[root@master dashboard]# kubectl get pods -n kube-system -owide
NAME                                    READY   STATUS    RESTARTS   AGE   IP            NODE     NOMINATED NODE   READINESS GATES
coredns-fb8b8dccf-bh4t2                 /     Running             14d   10.244.0.3    master   <none>           <none>
coredns-fb8b8dccf-rxkjw                 /     Running             14d   10.244.0.2    master   <none>           <none>
etcd-master                             /     Running             14d   10.0.1.195    master   <none>           <none>
heapster-5d4bf58946-bfvcz               /     Running             13m   10.244.2.28   node01   <none>           <none>
kube-apiserver-master                   /     Running             14d   10.0.1.195    master   <none>           <none>
kube-controller-manager-master          /     Running             14d   10.0.1.195    master   <none>           <none>
kube-flannel-ds-amd64-             /     Running             14d   10.0.1.229    node01   <none>           <none>
kube-flannel-ds-amd64-nhvm5             /     Running             14d   10.0.1.7      node02   <none>           <none>
kube-flannel-ds-amd64-xwng8             /     Running             14d   10.0.1.195    master   <none>           <none>
kube-proxy-2mk2h                        /     Running             14d   10.0.1.195    master   <none>           <none>
kube-proxy-54x8x                        /     Running             14d   10.0.1.7      node02   <none>           <none>
kube-proxy-n9rdx                        /     Running             14d   10.0.1.229    node01   <none>           <none>
kube-scheduler-master                   /     Running             14d   10.0.1.195    master   <none>           <none>
kubernetes-dashboard-769b74d4b4-kk6zx   /     Running             38m   10.244.2.27   node01   <none>           <none>
monitoring-grafana-658976d65f-xgp6v     /     Running             13m   10.244.1.18   node02   <none>           <none>
monitoring-influxdb-866db5f944-8tq74    /     Running             13m   10.244.1.19   node02   <none>           <none>
tiller-deploy-58f5d95b9c-x492l          /     Running             13d   10.244.2.3    node01   <none>           <none>
traefik-ingress-controller-f9bg6        /     Running             73m   10.244.1.16   node02   <none>           <none>
traefik-ingress-controller-x6tjl        /     Running             73m   10.244.2.26   node01   <none>           <none>
[root@master dashboard]# kubectl logs  heapster-5d4bf58946-bfvcz -n kube-system
E0614 ::05.004107        manager.go:] Error in scraping containers from kubelet:10.0.1.195:: failed to get all container stats from Kubelet URL "http://10.0.1.195:10255/stats/container/": Post http://10.0.1.195:10255/stats/container/: dial tcp 10.0.1.195:10255: getsockopt: connection refused
W0614 ::25.000648        manager.go:] Failed to get all responses in time (got /)
E0614 ::05.022570        manager.go:] Error in scraping containers from kubelet:10.0.1.229:: failed to get all container stats from Kubelet URL "http://10.0.1.229:10255/stats/container/": Post http://10.0.1.229:10255/stats/container/: dial tcp 10.0.1.229:10255: getsockopt: connection refused
W0614 ::25.000593        manager.go:] Failed to get all responses in time (got /)

//上面还有报错

1.2修改heapster.yaml 文件

#### 修改如下部分 #####
因为 kubelet 启用了 https 所以如下配置需要增加 https 端口
        - --source=kubernetes:https://kubernetes.default
修改为
        - --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true

1.3修改heapster-rbac.yaml文件

#### 修改为部分 #####
将 serviceAccount kube-system:heapster 与 ClusterRole system:kubelet-api-admin 绑定，授予它调用 kubelet API 的权限；
[root@master heapster]# cat heapster-rbac.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:heapster
subjects:
- kind: ServiceAccount
  name: heapster
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster-kubelet-api
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:kubelet-api-admin
subjects:
- kind: ServiceAccount
  name: heapster
  namespace: kube-system

2创建应用

[root@master heapster]# kubectl apply -f ./
[root@master heapster]# kubectl logs  heapster-5d9575b66b-t5zxt -n kube-system
I0614 ::09.353325        heapster.go:] /heapster --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true --sink=influxdb:http://monitoring-influxdb.kube-system.svc:8086
I0614 ::09.353365        heapster.go:] Heapster version v1.5.4
I0614 ::09.353623        configs.go:] Using Kubernetes client with master "https://kubernetes.default" and version v1
I0614 ::09.353648        configs.go:] Using kubelet port
I0614 ::09.367340        influxdb.go:] created influxdb sink with options: host:monitoring-influxdb.kube-system.svc: user:root db:k8s
I0614 ::09.367368        heapster.go:] Starting with InfluxDB Sink
I0614 ::09.367374        heapster.go:] Starting with Metric Sink
I0614 ::09.375501        heapster.go:] Starting heapster on port
I0614 ::05.067815        influxdb.go:] Created database "k8s" on influxDB server at "monitoring-influxdb.kube-system.svc:8086"

3.查看监控节点

[root@master heapster]# kubectl top nodes
NAME     CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%
master   112m         %     2464Mi          %
node01   59m          %     4356Mi          %
node02   47m          %     1644Mi          % 

4.访问

可以从kuberbetes-dashboard访问https://www.cnblogs.com/linyouyi/p/11001213.html

也可以把上面的应用单独暴露NodePort然后通过NodeIP:NodePort访问

更加详细玩转https://www.cnblogs.com/scode2/p/8810052.html

grafana.yaml

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: monitoring-grafana
  namespace: kube-system
spec:
  replicas:
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: grafana
    spec:
      containers:
      - name: grafana
        image: k8s.gcr.io/heapster-grafana-amd64:v5.0.4
        ports:
        - containerPort:
          protocol: TCP
        volumeMounts:
        - mountPath: /etc/ssl/certs
          name: ca-certificates
          readOnly: true
        - mountPath: /var
          name: grafana-storage
        env:
        - name: INFLUXDB_HOST
          value: monitoring-influxdb
        - name: GF_SERVER_HTTP_PORT
          value: ""
          # The following env variables are required to make Grafana accessible via
          # the kubernetes api-server proxy. On production clusters, we recommend
          # removing these env variables, setup auth for grafana, and expose the grafana
          # service using a LoadBalancer or a public IP.
        - name: GF_AUTH_BASIC_ENABLED
          value: "false"
        - name: GF_AUTH_ANONYMOUS_ENABLED
          value: "true"
        - name: GF_AUTH_ANONYMOUS_ORG_ROLE
          value: Admin
        - name: GF_SERVER_ROOT_URL
          # If you're only using the API Server proxy, set this value instead:
          # value: /api/v1/namespaces/kube-system/services/monitoring-grafana/proxy
          value: /
      volumes:
      - name: ca-certificates
        hostPath:
          path: /etc/ssl/certs
      - name: grafana-storage
        emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  labels:
    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
    # If you are NOT using this as an addon, you should comment out this line.
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: monitoring-grafana
  name: monitoring-grafana
  namespace: kube-system
spec:
  # In a production setup, we recommend accessing Grafana through an external Loadbalancer
  # or through a public IP.
  # type: LoadBalancer
  # You could also use NodePort to expose the service at a randomly-generated port
  # type: NodePort
  ports:
  - port:
    targetPort:
  selector:
    k8s-app: grafana

heapster-rbac.yaml

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:heapster
subjects:
- kind: ServiceAccount
  name: heapster
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster-kubelet-api
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:kubelet-api-admin
subjects:
- kind: ServiceAccount
  name: heapster
  namespace: kube-system

heapster.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: heapster
  namespace: kube-system
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: heapster
  namespace: kube-system
spec:
  replicas:
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: heapster
    spec:
      serviceAccountName: heapster
      containers:
      - name: heapster
        image: k8s.gcr.io/heapster-amd64:v1.5.4
        imagePullPolicy: IfNotPresent
        command:
        - /heapster
        - --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true
        - --sink=influxdb:http://monitoring-influxdb.kube-system.svc:8086
---
apiVersion: v1
kind: Service
metadata:
  labels:
    task: monitoring
    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
    # If you are NOT using this as an addon, you should comment out this line.
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: Heapster
  name: heapster
  namespace: kube-system
spec:
  ports:
  - port:
    targetPort:
  selector:
    k8s-app: heapster

influxdb.yaml

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: monitoring-influxdb
  namespace: kube-system
spec:
  replicas:
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: influxdb
    spec:
      containers:
      - name: influxdb
        image: k8s.gcr.io/heapster-influxdb-amd64:v1.5.2
        volumeMounts:
        - mountPath: /data
          name: influxdb-storage
      volumes:
      - name: influxdb-storage
        emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  labels:
    task: monitoring
    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
    # If you are NOT using this as an addon, you should comment out this line.
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: monitoring-influxdb
  name: monitoring-influxdb
  namespace: kube-system
spec:
  ports:
  - port:
    targetPort:
  selector:
    k8s-app: influxdb