rabbit_消费者

import pika
import json
import time
import os
import ast
import uuid
import time
import json
import hashlib

import redis
import pymysql

import logging
from logging import handlers

# 日志记录
class Logger(object):
    level_relations = {
        'debug': logging.DEBUG,
        'info': logging.INFO,
        'warning': logging.WARNING,
        'error': logging.ERROR,
        'crit': logging.CRITICAL
    }  # 日志级别关系映射

    def __init__(self, filename, level='info', when='D', backCount=3,
                 fmt='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'):
        self.logger = logging.getLogger(filename)
        format_str = logging.Formatter(fmt)  # 设置日志格式
        self.logger.setLevel(self.level_relations.get(level))  # 设置日志级别
        sh = logging.StreamHandler()  # 往屏幕上输出
        sh.setFormatter(format_str)  # 设置屏幕上显示的格式
        th = handlers.TimedRotatingFileHandler(filename=filename, when=when, backupCount=backCount,
                                               encoding='utf-8')  # 往文件里写入#指定间隔时间自动生成文件的处理器
        # 实例化TimedRotatingFileHandler
        # interval是时间间隔，backupCount是备份文件的个数，如果超过这个个数，就会自动删除，when是间隔的时间单位，单位有以下几种：
        # S 秒
        # M 分
        # H 小时、
        # D 天、
        # W 每星期（interval==0时代表星期一）
        # midnight 每天凌晨
        th.setFormatter(format_str)  # 设置文件里写入的格式
        self.logger.addHandler(sh)  # 把对象加到logger里
        self.logger.addHandler(th)

file = 'all'
log = Logger('clear' + os.sep + '%s.log' % file)
logger = log.logger

DEBUG = False

class ClearLog(object):
    if DEBUG:
        DATABASE = 'unionlog'
        # 本地测试
        poll = redis.ConnectionPool(host='192.168.10.10', port=7000, db=5, password='', decode_responses=True)
        conn = pymysql.connect(host='192.168.10.5', user='root',
                               password='root',
                               database=DATABASE, charset='utf8')
        cursor = conn.cursor()
    else:
        DATABASE = 'log'
        # 线上正式
        poll = redis.ConnectionPool(host='192.168.5.219', port=6379, db=5, password='', decode_responses=True)
        conn = pymysql.connect(host='', user='datacenter',
                               password='kbs11zx@',
                               database=DATABASE, charset='utf8')
        cursor = conn.cursor()
    CONN = redis.Redis(connection_pool=poll)
    REDIS_PID_HASH = "tarsier.log.clear.pid.hash"
    REDIS_PID_DELETE_HASH = "tarsier.log.delete.pid.hash"
    REDIS_PID_DELETE_LIST = "tarsier.log.delete.pid.list"
    REDIS_PID_DELETE_LIST_TEMP = "tarsier.log.delete.pid.list.temp"
    table_list = []
    table = 'tarsier_log_details'
    instance = None

    def __new__(cls, *args, **kwargs):
        if cls.instance:
            return cls.instance
        else:
            return super().__new__(cls)

    @staticmethod
    def get_table_list(table):
        ClearLog.table = table
        # 判断表是否存在
        if table in ClearLog.table_list:
            # print('表存在1')
            pass
        else:
            ClearLog.cursor.execute("SHOW TABLES")
            res = ClearLog.cursor.fetchall()
            table_temp = []
            for i in res:
                table_temp.append(i[0])
            # print(table_temp)
            ClearLog.table_list = table_temp
            if table in ClearLog.table_list:
                # print('表存在2')
                pass
            else:
                # 创建表
                sql = """create table %s like tarsier_log_details""" % (table)
                try:
                    print('创建表')
                    ClearLog.cursor.execute(sql)
                except Exception as e:
                    pass
                ClearLog.cursor.execute("SHOW TABLES")
                res = ClearLog.cursor.fetchall()
                table_temp = []
                for i in res:
                    table_temp.append(i[0])
                ClearLog.table_list = table_temp

    # 更新数据库
    @staticmethod
    def updata_db(data):
        # ##################### 表名 #####################
        table = "tarsier_log_details_%s" % ClearLog.timestamp_to_str(format="%Y%m")

        ClearLog.get_table_list(table)
        keys = ', '.join(data.keys())
        values = ', '.join(['%s'] * len(data))
        # 实际用的是插入语句，不过加了ON DUPLICATE KEY UPDATE（主键存在，则执行更新操作）
        sql = 'INSERT INTO {table}({keys}) VALUES ({values}) ON DUPLICATE KEY UPDATE'.format(table=table, keys=keys,
                                                                                             values=values)
        update = ','.join([" {key} = %s".format(key=key) for key in data])
        sql += update
        try:
            ClearLog.cursor.execute(sql, tuple(data.values()) * 2)
            print('update Successful')
            ClearLog.conn.commit()
        except Exception as e:
            logger.error(e)
            print('update Failed')

    @staticmethod
    def update_db_sql(sql):
        try:
            ClearLog.cursor.execute(sql)  # 执行sql
            ClearLog.conn.commit()  # 提交到数据库
            print('更新成功')
        except Exception as e:
            print("ERROR:{}".format(str(e)))
            ClearLog.conn.rollback()  # 发生错误则回滚
            logger.info('error:%s' % str(e))

    def __call__(self, *args, **kwargs):
        pass

    def __init__(self):
        # ClearLog.main()
        pass

    @staticmethod
    def md5_me(key):
        md5 = hashlib.md5()
        md5.update(str(key).encode('utf-8'))
        value = md5.hexdigest()
        return value

    @staticmethod
    def main():
        with open('20201110.log', encoding='utf-8') as f:
            count = 0
            for item in f:
                line = item.strip()
                data = ast.literal_eval(line)
                # 数据清洗 - 开始
                ClearLog.clear_log(data)
                # print(ClearLog.__dict__)
                count += 1
                if count % 10000 == 0:
                    print(count)
                    # break

    @staticmethod
    def main2(data):
        # 数据清洗 - 开始
        ClearLog.clear_log(data)

    @staticmethod
    def clear_log(data):
        res_data = {}
        rsUid = data.get('rsUid', '')
        rsPageId = data.get('rsPageId', '')
        rshyuid = data.get('rshyuid', '')
        pageid = data.get('pageid', '')
        pageUrl = data.get('pageUrl', '')
        userAgent = data.get('userAgent', '')
        referrer = data.get('referrer', '')
        nowDate = data.get('nowDate', '')
        device = data.get('device', '')
        rshyType = data.get('rshyType', '')
        targetDataset = str(data.get('targetDataset', ''))
        targetValue = data.get('targetValue', '')
        targetClassName = data.get('targetClassName', '')
        inputData = str(data.get('inputData', ''))
        rshyUserIp = data.get('rshyUserIp', '')
        netloc = data.get('netloc', '')
        urlPath = data.get('urlPath', '')
        siteName = data.get('siteName', '')
        TIME = ClearLog.timestamp_to_str()
        ID = ClearLog.get_uuid()
        rshyTime = data.get('rshyTime', '')
        try:
            rsdate = rshyTime.split()[0]
            temp = rshyTime.split()[1]
            rshour = temp.split(':')[0]
        except:
            rsdate = ''
            rshour = 0
        res_data.update({
            "id": ID,
            "rsuid": rsUid,
            "rshytime": rshyTime,
            "rshour": rshour,
            "rsdate": rsdate,
            "rspageid": rsPageId,
            "rshyuid": rshyuid,
            "pageid": pageid,
            "pageurl": pageUrl,
            "useragent": userAgent,
            "referrer": referrer,
            "device": device,
            "rshytype": rshyType,
            "targetvalue": targetValue,
            "targetdataset": targetDataset,
            "targetclassname": targetClassName,
            "inputdata": inputData,
            "starttime": nowDate,
            "rshyuserip": rshyUserIp,
            "netloc": netloc,
            "urlpath": urlPath,
            "sitename": siteName,
            "createtime": TIME,
            "updatetime": TIME,
        })
        if rshyType == 'view' or rshyType == '':
            # 先判断这个值是否与存储一样
            rsUidKey = rsPageId  # ClearLog.md5_me(pageid)
            # print("pid", rsUidKey)
            if not rsPageId:
                return

            # 一直刷新pid
            ClearLog.CONN.hset(ClearLog.REDIS_PID_DELETE_HASH, rsUidKey, nowDate)

            res_temp = rsUid + pageUrl + referrer + userAgent + device
            # print('##############')
            res_rs_uid = ClearLog.md5_me(res_temp)
            # print(res_rs_uid)
            # 从redis中获取uid对应数据， 如果数据一样不做存储
            exist_uid = ClearLog.CONN.hget(ClearLog.REDIS_PID_HASH, rsUidKey)
            # print(exist_uid)
            if not exist_uid or res_rs_uid != str(exist_uid):
                ClearLog.CONN.hset(ClearLog.REDIS_PID_HASH, rsUidKey, res_rs_uid)
                # 数据入库
                ClearLog.write_data(res_data)
                # 存储一份记录时间hash
                ClearLog.CONN.hset(ClearLog.REDIS_PID_DELETE_HASH, rsUidKey, nowDate)
                # 并将此数据入删除队列
                data_temp = {"pid": rsUidKey, "date": nowDate}
                ClearLog.CONN.lpush(ClearLog.REDIS_PID_DELETE_LIST, json.dumps(data_temp))
            return
        # if not rshyType:
        #     return
        ClearLog.write_data(res_data)

    @staticmethod
    def write_data(data):
        ClearLog.updata_db(data)
        file_name = ClearLog.timestamp_to_str_m()
        with open('clear{}{}.clear.log'.format(os.sep, file_name), 'a+', encoding='utf-8') as f:
            f.write(str(data) + '\n')

    # 格式化时间转时间戳
    @staticmethod
    def str_to_timestamp(str_time=None, format='%Y-%m-%d %H:%M:%S'):
        if str_time:
            time_tuple = time.strptime(str_time, format)  # 把格式化好的时间转换成元祖
            result = time.mktime(time_tuple)  # 把时间元祖转换成时间戳
            return int(result)
        return int(time.time())

    # 把时间戳转换成格式化
    @staticmethod
    def timestamp_to_str(timestamp=None, format='%Y-%m-%d %H:%M:%S'):
        if timestamp:
            time_tuple = time.localtime(timestamp)  # 把时间戳转换成时间元祖
            result = time.strftime(format, time_tuple)  # 把时间元祖转换成格式化好的时间
            return result
        else:
            return time.strftime(format)

    # 把时间戳转换成格式化
    @staticmethod
    def timestamp_to_str_m(timestamp=None, format='%Y-%m-%d'):
        if timestamp:
            time_tuple = time.localtime(timestamp)  # 把时间戳转换成时间元祖
            result = time.strftime(format, time_tuple)  # 把时间元祖转换成格式化好的时间
            return result
        else:
            return time.strftime(format)

    # uuid
    @staticmethod
    def get_uuid():
        res = str(uuid.uuid4())
        UUID = ''.join(res.split('-'))
        return UUID

    # 每5分钟删除一次hash中的值，并将停留时间算出
    @staticmethod
    def del_tarsier_log_pid_hash():
        table = ClearLog.table + '_%s' % ClearLog.timestamp_to_str_m(format='%Y%m')

        print('每5分钟删除一次hash中的值，并将停留时间算出')
        get_pid_list = ClearLog.CONN.hgetall(ClearLog.REDIS_PID_DELETE_HASH)
        # print(get_pid_list)
        for hash_pid_item in get_pid_list:

            redisDate = ClearLog.CONN.hget(ClearLog.REDIS_PID_DELETE_HASH, hash_pid_item)
            # 如果存储时间与当前时间相差1min之外更新  最后访问时间与停留时间 并将hash的值进行删除  否则不做处理
            try:
                redis_data_time = ClearLog.str_to_timestamp(redisDate)
                now_data_time = time.time()
                chufatime = now_data_time - redis_data_time
                # starttime =
                # staytime = redis_data_time - starttime
                if chufatime >= 60:
                    # 进行更新操作
                    sql = """update {} set endtime='{}' where rspageid='{}'""".format(table, redisDate,
                                                                                      hash_pid_item)
                    print(sql)
                    ClearLog.update_db_sql(sql)
                    # 更新完进行redis的值删除操作
                    ClearLog.CONN.hdel(ClearLog.REDIS_PID_DELETE_HASH, hash_pid_item)
            except Exception as e:
                pass
        print('====================================')

    # 每一天清除一次队列信息
    @staticmethod
    def del_tarsier_log_pid_list():
        logger.info('每一天清除一次队列信息')
        res_str = ClearLog.CONN.lpop(ClearLog.REDIS_PID_DELETE_LIST)
        while res_str:
            try:
                # 并将此数据入删除队列
                # data_temp = {"pid": rsUidKey, "date": nowDate}
                res_json = json.loads(res_str)
                # print(res_json)
                nowDate = res_json.get("date", '')
                rsUidKey = res_json.get("pid", '')
                redis_data_time = ClearLog.str_to_timestamp(nowDate)
                now_data_time = time.time()
                chufatime = now_data_time - redis_data_time
                if chufatime >= 24 * 60 * 60:
                    # 更新完进行redis的值删除操作
                    ClearLog.CONN.hdel(ClearLog.REDIS_PID_HASH, rsUidKey)
                    # print('删除')
                else:
                    ClearLog.CONN.rpush(ClearLog.REDIS_PID_DELETE_LIST_TEMP, json.dumps(res_json))
                res_str = ClearLog.CONN.lpop(ClearLog.REDIS_PID_DELETE_LIST)
            except:
                pass
        # print('处理队列')
        res_str = ClearLog.CONN.lpop(ClearLog.REDIS_PID_DELETE_LIST_TEMP)
        while res_str:
            res_json = json.loads(res_str)
            ClearLog.CONN.rpush(ClearLog.REDIS_PID_DELETE_LIST, json.dumps(res_json))
            res_str = ClearLog.CONN.lpop(ClearLog.REDIS_PID_DELETE_LIST_TEMP)
        logger.info('清除完毕')

# 把时间戳转换成格式化
# 天
def timestamp_to_str_day(timestamp=None, format='%Y%m%d'):
    if timestamp:
        time_tuple = time.localtime(timestamp)  # 把时间戳转换成时间元祖
        result = time.strftime(format, time_tuple)  # 把时间元祖转换成格式化好的时间
        return result
    else:
        return time.strftime(format)

# Connect to RabbitMQ and create channel
rabbit_host = "192.168.2.129"
rabbit_username = 'rshy'
rabbit_password = 'root1234@AWJSW'
queue_topic = 'logs.collect.statistics'
user = pika.PlainCredentials(rabbit_username, rabbit_password)
connection = pika.BlockingConnection(pika.ConnectionParameters(host=rabbit_host, credentials=user,)) #  heartbeat=0
# connection = pika.BlockingConnection(pika.ConnectionParameters(host='192.168.2.129'))
channel = connection.channel()

# Declare and listen queue
# channel.queue_declare(queue=cfg.QUEUE_TOPIC)
channel.queue_declare(queue=queue_topic)

ClearLogObj = ClearLog()
def consumer():

    print(' [*] Waiting for messages. To exit press CTRL+C')

    # Function process and print data
    def callback(ch, method, properties, body):
        # print("Method: {}".format(method))
        # print("Properties: {}".format(properties))

        data = json.loads(body)
        # print("ID: {}".format(data['id']))
        # print("Name: {}".format(data['name']))
        # print('Description: {}'.format(data['description']))
        print("--data--:", data)
        ClearLogObj.main2(data)
        file_name = timestamp_to_str_day()
        with open('consumer' + os.sep + file_name + '.log', 'a+', encoding='utf-8') as f:
            f.write(str(data) + '\n')

    # Listen and receive data from queue
    # channel.basic_consume(cfg.QUEUE_TOPIC, callback, True)
    channel.basic_consume(queue_topic, callback, True)
    channel.start_consuming()

if __name__ == '__main__':
    consumer()