python日志logging配置

python日志logging配置

• 为了方便ELK收集日志，将日志打印成json格式
• 开发过程中，使用json格式不方便排查问题
• 本文章使用python的logging模块，一步步增加配置，来说明每个组件作用

原始日志

• python可以使用两种方式打印
• python默认打印级别为'WARNING'

import logging

# 直接使用logging打印日志
logging.info('info')  # 未打印
logging.warning('warning')  # WARNING:root:warning
logging.error('error')  # ERROR:root:error

# 使用logger打印日志
logger = logging.getLogger('demo')
logger.info('info')  # 未打印
logger.warning('warning')  # WARNING:demo:warning
logger.error('error')  # ERROR:demo:error

使用字典配置

• version为固定字段，目前为1
• 将级别设置为'INFO'

import logging
import logging.config

def use_config():
    """配置日志行为"""
    config = {"version": 1, "root": {"level": "INFO"}}
    logging.config.dictConfig(config)

use_config()

# 直接使用logging打印日志
logging.info('info')  # INFO:root:info
logging.warning('warning')  # WARNING:root:warning
logging.error('error')  # ERROR:root:error

# 使用logger打印日志
logger = logging.getLogger('demo')
logger.info('info')  # INFO:demo:info
logger.warning('warning')  # WARNING:demo:warning
logger.error('error')  # ERROR:demo:error

记录器

• "loggers"为指定记录器
• "root"实质上也是记录器，python默认的记录器
  • logging直接打印日志，相当于logging.getLogger('root')
  • 没有找到记录器名字的使用默认记录器（即'root'）

import logging
import logging.config

def use_config():
    """配置日志行为"""
    config = {
        "version": 1,
        "root": {
            "level": "WARNING"
        },
        "loggers": {
            "demo": {
                "level": "ERROR"
            }
        }
    }
    logging.config.dictConfig(config)

use_config()

# 直接使用logging打印日志
logging.info('info')  # 未打印
logging.warning('warning')  # WARNING:root:warning
logging.error('error')  # ERROR:root:error

# 使用logger打印日志
logger = logging.getLogger('demo')
logger.info('info')  # 未打印
logger.warning('warning')  # 未打印
logger.error('error')  # ERROR:demo:error

# 未找到记录器时，使用'root'记录器
logger2 = logging.getLogger('demo2')
logger2.info('info')  # 未打印
logger2.warning('warning')  # WARNING:demo2:warning
logger2.error('error')  # ERROR:demo2:error

处理器及格式化器

• "handlers"为处理器
  • 处理器有很多类型，可以打印屏幕、打印文件、发送邮件、发送http请求等，详情查看Useful Handlers官方文档
  • 记录器中可以指定多个处理器
  • 下面例子打印屏幕
• "formatters"为格式化器
  • format：使用python内置格式化器格式化日志信息

import logging
import logging.config

def use_config():
    """配置日志行为"""
    config = {
        "version": 1,
        "root": {
            "level": "INFO",
            "handlers": ["console_handler"]
        },
        "handlers": {
            "console_handler": {
                "class": "logging.StreamHandler",
                "formatter": "console_formatter",
                "stream": "ext://sys.stdout"
            }
        },
        "formatters": {
            "console_formatter": {
                "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
            }
        }
    }
    logging.config.dictConfig(config)

use_config()

logging.info('info')  # 2022-10-25 20:16:59,073 - root - INFO - info
logging.warning('warning')  # 2022-10-25 20:16:59,073 - root - WARNING - warning
logging.error('error')  # 2022-10-25 20:16:59,074 - root - ERROR - error

自定义格式化器

• 自定义格式化器继承logging.Formatter，重写format方法
• 配置中，字段名使用"class"代表自定义格式化器
  • 填入导入该类的字符串，如下代码，在"log_demo.py"文件里的"MyFormatter"，即为"log_demo.MyFormatter"
  • 类型导入会执行一遍代码，重复调用配置方法会添加多个处理器，导致打印多遍
  • record中有很多字段，这里不展示，自行打印查看

# log_demo.py
import logging
import logging.config

class MyFormatter(logging.Formatter):
    def format(self, record: logging.LogRecord) -> str:
        """重写日志格式化字符串"""
        return f"[{record.name}] [{record.levelname}] {record.msg}"

def use_config():
    """配置日志行为"""
    config = {
        "version": 1,
        "root": {
            "level": "INFO",
            "handlers": ["console_handler"]
        },
        "handlers": {
            "console_handler": {
                "class": "logging.StreamHandler",
                "formatter": "console_formatter",
                "stream": "ext://sys.stdout"
            }
        },
        "formatters": {
            "console_formatter": {
                "class": "log_demo.MyFormatter"
            }
        }
    }
    logging.config.dictConfig(config)

if __name__ == "__main__":
    """
    实例化MyFormatter类时，会执行一遍代码
    将代码放在__main__里，否则会打印两遍
    """
    use_config()

    logging.info('info') # [root] [INFO] info
    logging.warning('warning') # [root] [WARNING] warning
    logging.error('error') # [root] [ERROR] error

extra额外字段

• 日志方法中有"extra"字段，丰富日志信息
• 打印

import logging
import logging.config

class MyFormatter(logging.Formatter):
    def format(self, record: logging.LogRecord) -> str:
        """重写日志格式化字符串"""
        for i in record.__dict__:
            print(i, end=',')
        print('')
        return f"[{record.name}] [{record.levelname}] {record.msg}"

def use_config():
    """配置日志行为"""
    config = {
        "version": 1,
        "root": {
            "level": "INFO",
            "handlers": ["console_handler"]
        },
        "handlers": {
            "console_handler": {
                "class": "logging.StreamHandler",
                "formatter": "console_formatter",
                "stream": "ext://sys.stdout"
            }
        },
        "formatters": {
            "console_formatter": {
                "class": "log_demo.MyFormatter"
            }
        }
    }
    logging.config.dictConfig(config)

if __name__ == "__main__":
    """
    实例化MyFormatter类时，会执行一遍代码
    将代码放在__main__里，否则会打印两遍
    """
    use_config()

    logging.info('info')
    # name,msg,args,levelname,levelno,pathname,filename,module,exc_info,exc_text,stack_info,lineno,funcName,created,msecs,relativeCreated,thread,threadName,processName,process,
    # [root] [INFO] info

    # extra增加了'a','b'，可以看到record对象也增加了'a','b'两个字段
    logging.info('extra info', extra={'a': 1, 'b': 2})
    # name,msg,args,levelname,levelno,pathname,filename,module,exc_info,exc_text,stack_info,lineno,funcName,created,msecs,relativeCreated,thread,threadName,processName,process,a,b,
    # [root] [INFO] extra info

最终代码

• 增加环境变量"LOG_MODE"，用来控制打印行为
  • 为"json"时，打印json格式字符串，提供给ELK收集
  • 为"text"时，打印成文本模式，方便开发时查看
• json转换时加上"ensure_ascii=False"，否则中文会被转换为ASCII

代码

import os
import json
import logging
import logging.config
from datetime import datetime, timezone, timedelta

class JsonFormatter(logging.Formatter):
    __TZ = timezone(timedelta(hours=+8))

    def format(self, record: logging.LogRecord) -> str:
        # ELK收集需要使用带时区的时间戳，key必须为"@timestamp"
        create_time = datetime.fromtimestamp(record.created)
        kv = {'@timestamp': create_time.astimezone(self.__TZ).isoformat()}
        if len(record.args) > 0:
            kv['message'] = record.msg % record.args
        else:
            kv['message'] = record.msg
        # 报错信息
        if record.exc_info:
            kv['err_info'] = self.formatException(record.exc_info)

        for k in record.__dict__:
            v = getattr(record, k)
            # 其他类型json转换可能报错
            if v is None or isinstance(v, (int, float, bool, str)):
                kv[k] = v
            else:
                kv[k] = str(v)
        return json.dumps(kv, ensure_ascii=False)

def use_config():
    """配置日志行为"""
    if os.environ.get('LOG_MODE') == "json":
        format_data = {"class": "log_demo.JsonFormatter"}
    else:
        format_data = {
            "format": "[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s"
        }

    config = {
        "version": 1,
        "root": {
            "level": "INFO",
            "handlers": ["console_handler"]
        },
        "handlers": {
            "console_handler": {
                "class": "logging.StreamHandler",
                "formatter": "console_formatter",
                "stream": "ext://sys.stdout"
            }
        },
        "formatters": {
            "console_formatter": format_data
        }
    }

    logging.config.dictConfig(config)

if __name__ == "__main__":
    use_config()

    logging.info('info')
    logging.info('extra info', extra={'a': 1, 'b': 2})

text效果

# linux设置环境变量
export LOG_MODE=text
# windows设置环境变量
set LOG_MODE=text

python log_demo.py

[2022-10-26 19:11:07,468] [root] [INFO] info
[2022-10-26 19:11:07,468] [root] [INFO] extra info

json效果

# linux设置环境变量
export LOG_MODE=json
# windows设置环境变量
set LOG_MODE=json

python log_demo.py

{"@timestamp": "2022-10-26T20:10:42.168191+08:00", "message": "info", "name": "root", "msg": "info", "args": "()", "levelname": "INFO", "levelno": 20, "pathname": "e:\\Work\\Code\\TestCode\\everything\\log_demo.py", "filename": "log_demo.py", "module": "log_demo", "exc_info": null, "exc_text": null, "stack_info": null, "lineno": 66, "funcName": "<module>", "created": 1666786242.168191, "msecs": 168.19095611572266, "relativeCreated": 35.9044075012207, "thread": 22716, "threadName": "MainThread", "processName": "MainProcess", "process": 19576}
{"@timestamp": "2022-10-26T20:10:42.169210+08:00", "message": "extra info", "name": "root", "msg": "extra info", "args": "()", "levelname": "INFO", "levelno": 20, "pathname": "e:\\Work\\Code\\TestCode\\everything\\log_demo.py", "filename": "log_demo.py", "module": "log_demo", "exc_info": null, "exc_text": null, "stack_info": null, "lineno": 67, "funcName": "<module>", "created": 1666786242.1692102, "msecs": 169.21019554138184, "relativeCreated": 36.92364692687988, "thread": 22716, "threadName": "MainThread", "processName": "MainProcess", "process": 19576, "a": 1, "b": 2}