python logging with yaml

Recently, I was made a service which can provide a simple way to get best model. so, i spent lot of time to read source code of auto-sklearn, auto-sklearn is an automated machine learning toolkit and a drop-in replacement for a scikit-learn estimator.

when I write my logging module I found the logging module of auto-sklearn which used yaml file as config, the config file：

 ---
 version: 1
 disable_existing_loggers: False
 formatters:
   simple:
     format: '[%(asctime)s: %(levelname)s] %(message)s'

 handlers:
   console:
     class: logging.StreamHandler
     level: WARNING
     formatter: simple
     stream: ext://sys.stdout

   file_handler:
     class: logging.FileHandler
     level: DEBUG
     formatter: simple
     filename: autosklearn.log

 root:
   level: ERROR
   handlers: [console, file_handler]

 loggers:
   server:
     level: INFO
     handlers: [file_handler]
     propagate: no

the caller can define loggers、handlers、formatters here, after that, caller need write setup code and get logger code.

 # -*- encoding: utf-8 -*-
 import logging
 import logging.config
 import os
 import yaml

 def setup_logger(output_file=None, logging_config=None):
     # logging_config must be a dictionary object specifying the configuration
     if not os.path.exists(os.path.dirname(output_file)):
         os.makedirs(os.path.dirname(output_file))
     if logging_config is not None:
         if output_file is not None:
             logging_config['handlers']['file_handler']['filename'] = output_file
         logging.config.dictConfig(logging_config)
     else:
         with open(os.path.join(os.path.dirname(__file__), 'logging.yaml'),
                   'r') as fh:
             logging_config = yaml.safe_load(fh)
         if output_file is not None:
             logging_config['handlers']['file_handler']['filename'] = output_file
         logging.config.dictConfig(logging_config)

 def _create_logger(name):
     return logging.getLogger(name)

 def get_logger(name):
     logger = PickableLoggerAdapter(name)
     return logger

 class PickableLoggerAdapter(object):

     def __init__(self, name):
         self.name = name
         self.logger = _create_logger(name)

     def __getstate__(self):
         """
         Method is called when pickle dumps an object.

         Returns
         -------
         Dictionary, representing the object state to be pickled. Ignores
         the self.logger field and only returns the logger name.
         """
         return {'name': self.name}

     def __setstate__(self, state):
         """
         Method is called when pickle loads an object. Retrieves the name and
         creates a logger.

         Parameters
         ----------
         state - dictionary, containing the logger name.

         """
         self.name = state['name']
         self.logger = _create_logger(self.name)

     def debug(self, msg, *args, **kwargs):
         self.logger.debug(msg, *args, **kwargs)

     def info(self, msg, *args, **kwargs):
         self.logger.info(msg, *args, **kwargs)

     def warning(self, msg, *args, **kwargs):
         self.logger.warning(msg, *args, **kwargs)

     def error(self, msg, *args, **kwargs):
         self.logger.error(msg, *args, **kwargs)

     def exception(self, msg, *args, **kwargs):
         self.logger.exception(msg, *args, **kwargs)

     def critical(self, msg, *args, **kwargs):
         self.logger.critical(msg, *args, **kwargs)

     def log(self, level, msg, *args, **kwargs):
         self.logger.log(level, msg, *args, **kwargs)

     def isEnabledFor(self, level):
         return self.logger.isEnabledFor(level)

get_logger return a logger object, setup_logger setup logger handler which define output where.

when you use it,like this

 from .util import get_logger, setup_logger

 APP_DIR = os.path.dirname(os.path.abspath(__file__))
 setup_logger(output_file=os.path.join(APP_DIR, 'logs', 'server.log'))
 LOG = get_logger("server")

Define a global variable LOG for other modules to call.

all done.