Python3对接华三CAS平台Api获取虚拟机监控信息-渐入佳境

--时间：2021年2月3日

--作者：飞翔的小胖猪

说明

使用python对接华三CAS虚拟化平台，通过厂商提供的api接口获取每个集群下所有虚拟机的监控信息，并保存数据在本地的mariadb数据库。

使用脚本时请读者根据自己的实际情况填入用户名、密码、cas平台地址。

设置线程时不要设置过多，脚本设置为20个在执行时明显能感觉到cas网页有点慢。

环境

华三CAS平台版本： V5.0(E0513H02)

Python版本：3.8.4

操作系统版本： Centos Linux release 7.5.1804

数据库版本：mariadb-server-5.5.65

数据库

创建数据库及用户

MariaDB [(none)] > create database cas_info;
MariaDB [(none)] > create user cas@'%'  identified by 'Cas_123456#'
MariaDB [(none)] > grant all privileges on cas_info.* to cas@'%';
MariaDB [(none)] > flush privileges;

数据表创建语句

create table cas_trend(
id int(10) primary key not null auto_increment,
vm_id varchar(50),
vm_ip varchar(30),
cpu_rate varchar(15),
mem_rate varchar(15),
vm_stat varchar(6),
net_r varchar(20),
net_r_p varchar(20),
net_w varchar(20),
net_w_p varchar(20),
io_mbps varchar(20),
io_r_mbps varchar(20),
io_w_mpbs varchar(20),
io_r_iops varchar(20),
io_w_iops varchar(20),
timestamp varchar(30)
);

脚本

from http.cookiejar import CookieJar,MozillaCookieJar
import numpy
import _thread
import threading
import requests
import time
import pymysql
import re
import sys
import lxml.html
import urllib3.request
import urllib.request
from http.cookiejar import CookieJar,MozillaCookieJar

def get_cas_session():
    #定义获取到cas的token值。
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36'}
    #其中的name和password值 可以手动使用浏览器访问一下cas然后去header里面截取出来。
    url='http://X.X.X.X:8080/cas/spring_check?encrypt=true&lang=cn&name=TESTDBAVsRM&password=HLLWA8ATwKqvLWn94opBBB3ynOBrVKXXX'
    cookiejar = CookieJar()
    handler = urllib.request.HTTPCookieProcessor(cookiejar)
    opener = urllib.request.build_opener(handler)
    data = {
    'encrypt':'true',
    'lang':'cn',
    'name':'TESTDBAVsRM',
    'password':'HLLWA8ATwKqvLWn94opBBB3ynOBrVKXXX'
    }
    request = urllib.request.Request(url,headers=headers)
    respose = opener.open(request,data=urllib.parse.urlencode(data).encode())
    return opener
    #完成cookie值获取完成。

def get_all_vm_id(opener):
    #获取地址池中的所有虚拟机id
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36'}
    pool_vm = list()
    aa=0
    #drl1:8    ty1:10   ty2:11  ty3:12   ty4:14
    pool_url="http://x.x.x.x:8080/cas/casrs/vm/vmList?hpId=14"
    pool_vm_list_request = urllib.request.Request(pool_url,headers=headers)
    pool_vm_list_respose = opener.open(pool_vm_list_request).read()
    pool_vm_list_respose_utf = pool_vm_list_respose.decode("utf-8","ignore")
    #pat_vm='<id>(.*?)</id>'
    pat_vm='<domain>*<id>(.*?)</id>'
    pool_all_vm_list=re.compile(pat_vm).findall(pool_vm_list_respose_utf)
    for j in pool_all_vm_list:
        pool_vm.append(j)
        aa=aa+1
    #print(pool_vm)
    print("一共有虚拟机:",aa)
    return pool_vm

#获取主机ip地址
#传入虚拟机ID号列表和opener数据。输出id和ip地址组成字典。
def get_vm_ip(pool_vm,opener):
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36'}
    machine_id_ip_dir =dict()
    pat_vm_ip='<ipAddr>(.*?)</ipAddr>'
    for vm_id in pool_vm:
        vm_info_url ='http://X.X.X.X:8080/cas/casrs/vm/network/'+str(vm_id)
        vm_info_request = urllib.request.Request(vm_info_url,headers=headers)
        vm_info_respose = opener.open(vm_info_request).read()
        vm_info_respose_utf = vm_info_respose.decode("utf-8","ignore")
        vm_info_ip=re.compile(pat_vm_ip).findall(vm_info_respose_utf)
        #print("this id number:",vm_id)
        if len(vm_info_ip) == 0:
            machine_id_ip_dir[vm_id] = 'not values'
            #print("this id machine ip null,not install tools .")
        else:
            machine_id_ip_dir[vm_id] = vm_info_ip[0]
            #print("this machine ip address:",vm_info_ip[0])
    #print("this id_ip_dir values:",machine_id_ip_dir);
    return machine_id_ip_dir

#获取主机特定的信息
def get_vm_info(vm_id,opener):
    #定义一个保存虚拟机信息的列表
    vm_info_n_dict=dict()
    global dict_vm_info
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36'}
    #定义正则匹配规则
    pat_vm_cpu='<cpuRate>(.*?)</cpuRate>'
    pat_vm_mem='<memRate>(.*?)</memRate>'
    pat_vm_stat='<status>(.*?)</status>'
    pat_vm_uptime='<uptime>(.*?)</uptime>'
    pat_vm_net_r='<net>.*<readFlow>(.*?)</readFlow>'
    pat_vm_net_r_pack='<net>.*<readPackets>(.*?)</readPackets>'
    pat_vm_net_w='<net>.*<writeFlow>(.*?)</writeFlow>'
    pat_vm_net_w_pack='<net>.*<writePackets>(.*?)</writePackets>'

    #定义获取磁盘信息正则匹配规则
    pat_vm_disk_Mbps='<disk>.*<ioStat>(.*?)</ioStat>'
    pat_vm_disk_r_Mbps='<disk>.*<readStat>(.*?)</readStat>'
    pat_vm_disk_w_Mbps='<disk>.*<writeStat>(.*?)</writeStat>'
    pat_vm_disk_r_ps='<disk>.*<readReqest>(.*?)</readReqest>'
    pat_vm_disk_w_ps='<disk>.*<writeReqest>(.*?)</writeReqest>'

    #获取指定的ID的虚拟机的监控信息。
    vm_xx_info_url = 'http://X.X.X.X:8080/cas/casrs/vm/id/'+str(vm_id)+'/monitor'
    vm_xx_info_request = urllib.request.Request(vm_xx_info_url,headers=headers)
    vm_xx_info_respose = opener.open(vm_xx_info_request).read()
    vm_xx_info_respose_utf = vm_xx_info_respose.decode("utf-8","ignore")

    #正则表达式匹配,定义的数据。
    #cpu及内存状态启动时间等数据
    vm_info_cpu=re.compile(pat_vm_cpu).findall(vm_xx_info_respose_utf)
    vm_info_mem=re.compile(pat_vm_mem).findall(vm_xx_info_respose_utf)
    vm_info_stat=re.compile(pat_vm_stat).findall(vm_xx_info_respose_utf)
    vm_info_uptime=re.compile(pat_vm_uptime).findall(vm_xx_info_respose_utf)
    #主机网卡信息
    vm_info_net_r=re.compile(pat_vm_net_r).findall(vm_xx_info_respose_utf)
    vm_info_net_r_p=re.compile(pat_vm_net_r_pack).findall(vm_xx_info_respose_utf)
    vm_info_net_w=re.compile(pat_vm_net_w).findall(vm_xx_info_respose_utf)
    vm_info_net_w_p=re.compile(pat_vm_net_w_pack).findall(vm_xx_info_respose_utf)
    #磁盘信息
    vm_info_disk_Mbps=re.compile(pat_vm_disk_Mbps).findall(vm_xx_info_respose_utf)
    vm_info_disk_r_Mbps=re.compile(pat_vm_disk_r_Mbps).findall(vm_xx_info_respose_utf)
    vm_info_disk_w_Mbps=re.compile(pat_vm_disk_w_Mbps).findall(vm_xx_info_respose_utf)
    vm_info_disk_r_ps=re.compile(pat_vm_disk_r_ps).findall(vm_xx_info_respose_utf)
    vm_info_disk_w_ps=re.compile( pat_vm_disk_w_ps).findall(vm_xx_info_respose_utf)    

    #print("单个字典为",vm_info_n_dict)
    #判断如果匹配到多个值，就只获取第一个值哈哈哈哈。简单才是王道 好耶
    #赋值CPU信息
    if len(vm_info_cpu) > 1:
        vm_info_n_dict['cpu_rate']=vm_info_cpu[0]
    else:
        vm_info_n_dict['cpu_rate']=vm_info_cpu
    #赋值内存信息
    if len(vm_info_mem) > 1:
        vm_info_n_dict['mem_rate']=vm_info_mem[0]
    else:
        vm_info_n_dict['mem_rate']=vm_info_mem
    #赋值主机状态信息
    if len(vm_info_stat) > 1:
        vm_info_n_dict['vm_stat']=vm_info_stat[0]
    else:
        vm_info_n_dict['vm_stat']=vm_info_stat
    #赋值网络读流量值
    if len(vm_info_net_r) > 1:
        vm_info_n_dict['net_r']=vm_info_net_r[0]
    else:
        vm_info_n_dict['net_r']=vm_info_net_r
    #赋值网络读包数量
    if len(vm_info_net_r_p) > 1:
        vm_info_n_dict['net_r_p']=vm_info_net_r_p[0]
    else:
        vm_info_n_dict['net_r_p']=vm_info_net_r_p
    #赋值网络写流量
    if len(vm_info_net_w) > 1:
        vm_info_n_dict['net_w']=vm_info_net_w[0]
    else:
        vm_info_n_dict['net_w']=vm_info_net_w
    #赋值网络包写数量
    if len(vm_info_net_w_p) > 1:
        vm_info_n_dict['net_w_p']=vm_info_net_w_p[0]
    else:
        vm_info_n_dict['net_w_p']=vm_info_net_w_p
    #赋值磁盘总吞吐量
    if len(vm_info_disk_Mbps) > 1:
        vm_info_n_dict['io_mbps']=vm_info_disk_Mbps[0]
    else:
        vm_info_n_dict['io_mbps']=vm_info_disk_Mbps
    #赋值磁盘读速率
    if len(vm_info_disk_r_Mbps) > 1:
        vm_info_n_dict['io_r_mbps']=vm_info_disk_r_Mbps[0]
    else:
        vm_info_n_dict['io_r_mbps']=vm_info_disk_r_Mbps
    #赋值磁盘写速率
    if len(vm_info_disk_w_Mbps) > 1:
        vm_info_n_dict['io_w_mpbs']=vm_info_disk_w_Mbps[0]
    else:
        vm_info_n_dict['io_w_mpbs']=vm_info_disk_w_Mbps
    #赋值磁盘读ps
    if len(vm_info_disk_r_ps) > 1:
        vm_info_n_dict['io_r_iops']=vm_info_disk_r_ps[0]
    else:
        vm_info_n_dict['io_r_iops']=vm_info_disk_r_ps
    #赋值磁盘写ps
    if len(vm_info_disk_w_ps) > 1:
        vm_info_n_dict['io_w_iops']=vm_info_disk_w_ps[0]
    else:
        vm_info_n_dict['io_w_iops']=vm_info_disk_w_ps
    #把单个字典以vm_id为key加入到全局字典中。
    dict_vm_info[vm_id]=vm_info_n_dict
    #print("测试全局变量：",dict_vm_info)

def main():
    #主函数调用
    #global dict_vm_info
    #dict_vm_info['111']="test1111"
    opener = get_cas_session()
    pool_vm = get_all_vm_id(opener)
    id_ip_dict =  get_vm_ip(pool_vm,opener)
    #print("主机id号字典:",pool_vm)
    #print("主机ip地址字典：",id_ip_dict)

    #测试多线程
    str_len = len(pool_vm)
    #定义线程数量，不要超过str_len字符串长度
    thr_num=20
    dict1=dict()
    thread_num(str_len,thr_num,pool_vm,opener)
    print('数据获取完毕，关闭opener连接..........................')
    opener.close()
    print('整合获取到的数据并入库................................')
    #print("最后的字典是：", dict_vm_info)
    into_vminfo_db(pool_vm,id_ip_dict,dict_vm_info)
    print('cas平台虚拟机监控信息采集入库完成。')

def func(arg,test_str,opener):
    #线程掉用子函数
    for i in arg:
        #print(threading.currentThread().name, "----", i+1)
        print("实际值为：",test_str[i])
        get_vm_info(test_str[i],opener)

def thread_num(total,num,test_str,opener):
    #线程执行程序函数
    result=numpy.array_split(range(total),num) #分段
    for  i in result:
        #print("此时i的值为：",i)
        t=threading.Thread(target=func,args=(i,test_str,opener,)) #声明线程
        t.start() #启动线程
    while threading.active_count()!=1: #等待子线程
        pass

def into_vminfo_db(pool_vm,id_ip_dict,vm_info_dict):
    #数据入库函数
    ticks = time.time()
    db = pymysql.connect("127.0.0.1","cas","Cas_123456#","cas_info")
    cursor = db.cursor()
    for i in pool_vm:
        sql="insert into  cas_trend(vm_id,vm_ip,cpu_rate,mem_rate,vm_stat,net_r,net_r_p,net_w,net_w_p,io_mbps,io_r_mbps,io_w_mpbs,io_r_iops,io_w_iops,timestamp) values(\'" + str(i) +"\',\'"  + str(id_ip_dict[i]) +"\',\'"+ str(dict_vm_info[i]["cpu_rate"]).lstrip('[\'').rstrip('\']') +"\',\'"+ str(dict_vm_info[i]["mem_rate"]).lstrip('[\'').rstrip('\']') +"\',\'"+ str(dict_vm_info[i]["vm_stat"]).lstrip('[\'').rstrip('\']') +"\',\'"+str(dict_vm_info[i]["net_r"]).lstrip('[\'').rstrip('\']') +"\',\'"+str(dict_vm_info[i]["net_r_p"]).lstrip('[\'').rstrip('\']') +"\',\'"+str(dict_vm_info[i]["net_w"]).lstrip('[\'').rstrip('\']') +"\',\'"+ str(dict_vm_info[i]["net_w_p"]).lstrip('[\'').rstrip('\']') +"\',\'"+ str(dict_vm_info[i]["io_mbps"]).lstrip('[\'').rstrip('\']') +"\',\'"+str(dict_vm_info[i]["io_r_mbps"]).lstrip('[\'').rstrip('\']') +"\',\'"+str(dict_vm_info[i]["io_w_mpbs"]).lstrip('[\'').rstrip('\']') +"\',\'"+str(dict_vm_info[i]["io_r_iops"]).lstrip('[\'').rstrip('\']') +"\',\'"+str(dict_vm_info[i]["io_w_iops"]).lstrip('[\'').rstrip('\']')  +"\',\'"+ str(ticks) + "\')"
        print("此时sql语句为：",sql)
        try:
            cursor.execute(sql)
            db.commit()
        except:
            db.rollback()
    db.close()

if __name__ == "__main__":
    dict_vm_info=dict()
    lock = threading.Lock()
    count = 0
    main()

结果

命令执行结果

数据库结果展示