prometheus 监控系统

• 一. 安装docker环境
• 二. 安装prometheus
  • 2.1 编辑配置文件
  • 2.2 编辑docker-compose
• 三. grafana 展示
• 四 添加监控节点
• 五. 监控 java进程
• 六. 自动化监控服务

一. 安装docker环境

#01 安装docker

# step 1: 安装必要的一些系统工具

sudo yum install -y yum-utils device-mapper-persistent-data lvm2

# Step 2: 添加软件源信息

sudo yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

# Step 3

sudo sed -i 's+download.docker.com+mirrors.aliyun.com/docker-ce+' /etc/yum.repos.d/docker-ce.repo

# Step 4: 更新并安装Docker-CE

sudo yum makecache fast

sudo yum -y install docker-ce

# Step 4: 开启Docker服务

sudo service docker start

 

#02 设置镜像加速

sudo mkdir -p /etc/docker

sudo tee /etc/docker/daemon.json <<-'EOF'

{

  "registry-mirrors": ["https://9akof579.mirror.aliyuncs.com"]

}

EOF

sudo systemctl daemon-reload

sudo systemctl restart docker

 

#03 安装 docker-compose

sudo curl -L "https://github.com/docker/compose/releases/download/1.25.5/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose

sudo chmod +x /usr/local/bin/docker-compose

二. 安装prometheus

2.1 编辑配置文件

#01 编辑配置文件 邮件相关

1) alertmanager.yaml

[root@iZ2ze9nv7e19wvoseb2sl1Z prometheus]# cat alertmanager.yaml

global:

  smtp_smarthost: 'z15119911990@163.com:25'　　#163服务器

  smtp_from: '522308001@qq.com'　　　　　　　　#发邮件的邮箱

  smtp_auth_username: 'z15119911990'　　#发邮件的邮箱用户名，也就是你的邮箱

  smtp_auth_password: 'RVDDRXWLPQHVYNOB'#发邮件的邮箱密码

  smtp_require_tls: false　　　　　　　　#不进行tls验证

 

route:

  group_by: ['alertname']

  group_wait: 10s

  group_interval: 10s

  repeat_interval: 10m

  receiver: live-monitoring

 

receivers:

- name: 'live-monitoring'

  email_configs:

  - to: 'z15119911990@163.com'

 

2) node_down.yml 自定义的告警规则文件

[root@iZ2ze9nv7e19wvoseb2sl1Z prometheus]# cat node_down.yml

groups:

- name: node_down

  rules:

  - alert: InstanceDown

    expr: up == 0

    for: 1m

    labels:

      user: test

    annotations:

      summary: "Instance {{ $labels.instance }} down"

      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes."

 

 

3）主配置文件

[root@iZ2ze9nv7e19wvoseb2sl1Z prometheus]# grep -Ev '^#|^$' prometheus.yml

global:

  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.

  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.

  # scrape_timeout is set to the global default (10s).

alerting:

  alertmanagers:

  - static_configs:

    - targets: ['172.17.24.204:9093']

      # - alertmanager:9093

rule_files:

  - "node_down.yml"

  # - "first_rules.yml"

  # - "second_rules.yml"

scrape_configs:

  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.

  - job_name: 'prometheus'

    static_configs:

    - targets: ['172.17.24.204:9090']

  - job_name: 'cadvisor'

    static_configs:

    - targets: ['172.17.24.204:8080']

  - job_name: 'node'

    scrape_interval: 8s

    static_configs:

      - targets: ['172.17.24.204:9100','172.17.24.203:9100']    #抓取被监控主机IP

2.2 编辑docker-compose

#01 书写docker-compose

[root@iZ2ze9nv7e19wvoseb2sl1Z prometheus]# cat docker-compose.yaml

version: '2'

 

networks:

    mynet:

        driver: bridge

 

services:

    prometheus:

        image: prom/prometheus

        container_name: prometheus

        hostname: prometheus

        restart: always

        volumes:

            - ./prometheus.yml:/etc/prometheus/prometheus.yml

            - ./node_down.yml:/etc/prometheus/node_down.yml

        ports:

            - "9090:9090"

        networks:

            - mynet

 

    alertmanager:

        image: prom/alertmanager

        container_name: alertmanager

        hostname: alertmanager

        restart: always

        volumes:

            - ./alertmanager.yaml:/etc/alertmanager/alertmanager.yaml

        ports:

            - "9093:9093"

        networks:

            - mynet

 

    grafana:

        image: grafana/grafana

        container_name: grafana

        hostname: grafana

        restart: always

        ports:

            - "3000:3000"

        networks:

            - mynet

 

    node-exporter:

        image: prom/node-exporter

        #image: quay.io/prometheus/node-exporter

        container_name: node-exporter

        hostname: node-exporter

        restart: always

        ports:

            - "9100:9100"

        networks:

            - mynet

    cadvisor:

        image: google/cadvisor:latest

        container_name: cadvisor

        hostname: cadvisor

        restart: always

        volumes:

            - /:/rootfs:ro

            - /var/run:/var/run:rw

            - /sys:/sys:ro

            - /var/lib/docker/:/var/lib/docker:ro

        ports:

            - "8080:8080"

        networks:

            - mynet

 

#02 启动 docker-compose up -d

 

#03 web验证

 

http://172.17.24.204:9090/

三. grafana 展示

#01 登入地址

172.17.24.204:3000

user: admin

passwd: admin

 

#02 使用node 模版

四 添加监控节点

• 选择对应的 exporter

#01 安装exporter

docker pull prom/node-exporter

docker run -d -p 9100:9100 --restart=always  prom/node-exporter

 

 

#02 访问验证

http://localhost:9100/

 

#03 修改服务端 prometheus配置文件

  - job_name: 'node'

    scrape_interval: 8s

    static_configs:

      - targets: ['172.17.24.204:9100','172.17.24.203:9100']    #抓取被监控主机IP

五. 监控 java进程

#01 下载

wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.17.0/jmx_prometheus_javaagent-0.17.0.jar

 

#02 书写配置文件

[root@iZ2ze9nv7e19wvoseb2skzZ prometheus]# cat simple-config.yml

lowercaseOutputLabelNames: true

lowercaseOutputName: true

whitelistObjectNames: ["java.lang:type=OperatingSystem"]

blacklistObjectNames: []

rules:

  - pattern: 'java.lang<type=OperatingSystem><>(committed_virtual_memory|free_physical_memory|free_swap_space|total_physical_memory|total_swap_space)_size:'

    name: os_$1_bytes

    type: GAUGE

    attrNameSnakeCase: true

  - pattern: 'java.lang<type=OperatingSystem><>((?!process_cpu_time)w+):'

    name: os_$1

    type: GAUGE

    attrNameSnakeCase: true

 

#03 启动服务

java -javaagent:./jmx_prometheus_javaagent-0.17.0.jar=9400:simple-config.yaml -jar <你的jar>

nohup java -javaagent:./jmx_prometheus_javaagent-0.17.0.jar=9400:simple-config.yml -jar /data/project/pigx-upms-biz/pigx-upms-biz.jar -java.tmp.dir=/data/upload_tmp &

 

 

#04 书写启动脚本

/opt/prometheus/jmx_prometheus_javaagent-0.17.0.jar

/opt/prometheus/simple-config.yml

-javaagent:/opt/prometheus/jmx_prometheus_javaagent-0.17.0.jar=9400:/opt/prometheus/simple-config.yml

 

[root@iZ2ze9nv7e19wvoseb2skzZ prometheus]# cat  /root/sh/pigx-upms-biz.sh

#!/bin/bash

source /etc/profile

PORT=4000

kill -9 $(lsof -i:$PORT | grep LISTEN | awk '{print $2}')

sleep 5s

echo "$name kill"

name_tmp=${0%\.*}

name=${name_tmp##*/}

LOG_PATH=${name%%.jar}

Promtheus="-javaagent:/opt/prometheus/jmx_prometheus_javaagent-0.17.0.jar=9400:/opt/prometheus/simple-config.yml "

echo $name

#nohup java -jar -Dfile.encoding=UTF-8 -Xms128m -Xmx1g -Dlog.path=/pigx/$LOG_PATH /data/project/$name/$name.jar -java.tmp.dir=/data/upload_tmp >> /data/project/$name/$name.log 2>&1 &

nohup   java ${Promtheus} -jar -Dfile.encoding=UTF-8 -Xms128m -Xmx1g -Dlog.path=/pigx/$LOG_PATH /data/project/$name/$name.jar -java.tmp.dir=/data/upload_tmp >> /data/project/$name/$name.log 2>&1 &

六. 自动化监控服务

• Java 项目集成的前提条件

• 需要是 springboot 的项目

• 需要启用 actuator

• 需要集成 micrometer

• 需要激活配置

• 参考链接：https://blog.csdn.net/chuanbo0429/article/details/100751862

#依赖项目

         <dependency>

               <groupId>io.micrometer</groupId>

               <artifactId>micrometer-registry-prometheus</artifactId>

           </dependency>

           <dependency>

               <groupId>org.springframework.boot</groupId>

               <artifactId>spring-boot-starter-actuator</artifactId>

               <version>2.2.0.RELEASE</version>

           </dependency>

• 启动类

@SpringBootApplication

public class Springboot2PrometheusApplication {

  

    public static void main(String[] args) {

        SpringApplication.run(Springboot2PrometheusApplication.class, args);

    }

    @Bean

    MeterRegistryCustomizer<MeterRegistry> configurer(

            @Value("${spring.application.name}") String applicationName) {

        return (registry) -> registry.config().commonTags("application", applicationName);

    }

}

• 配置

management.endpoints.web.exposure.include=*
management.metrics.tags.application=${spring.application.name}
micrometer-registry-prometheus 指定版本可能会导致不兼容

management.endpoints.web.base-path=/admin
management.server.port=8090

• grafana id = 4701