[SaltStack] salt-master启动流程

SaltStack源码阅读

做salt有一段时间了, 一直没从源码层面去理解, 好吧, 开始读读源码 -_-

---

那就从salt-master的启动开始吧.

启动salt-master方法:

/etc/init.d/salt-master start

看看/etc/init.d/salt-master逻辑:

$ cat /etc/init.d/salt-master

SALTMASTER=/usr/bin/salt-master
PYTHON=/usr/bin/python
MASTER_ARGS=""

start() {
	echo -n $"Starting salt-master daemon: "
	if [ -f $SUSE_RELEASE ]; then
		startproc -f -p /var/run/$SERVICE.pid $SALTMASTER -d $MASTER_ARGS
		rc_status -v
	elif [ -e $DEBIAN_VERSION ]; then
		if [ -f $LOCKFILE ]; then
			echo -n "already started, lock file found"
			RETVAL=1
		elif $PYTHON $SALTMASTER -d $MASTER_ARGS >& /dev/null; then
			echo -n "OK"
			RETVAL=0
		fi
	else
		daemon --check $SERVICE $SALTMASTER -d $MASTER_ARGS
	fi
	RETVAL=$?
	echo
	return $RETVAL
}

继续看看/usr/bin/salt-master:

$ cat /usr/bin/salt-master

#!/usr/bin/python
'''
Start the salt-master
'''

from salt.scripts import salt_master

if __name__ == '__main__':
	salt_master()

调用salt_master()方法, 在script.py里:

$ cat scripts.py

def salt_master():
'''
Start the salt master.
'''
import salt.cli.daemons
master = salt.cli.daemons.Master()
master.start()

这里调用了salt模块Master类的start方法, 类方法在: ~/salt/cli/daemons.py

class Master(parsers.MasterOptionParser):
'''
Creates a master server
'''
def prepare(self):
    '''
    Run the preparation sequence required to start a salt master server.

    If sub-classed, don't **ever** forget to run:

        super(YourSubClass, self).prepare()
    '''
    self.parse_args()

    try:
        if self.config['verify_env']:
            v_dirs = [
                    self.config['pki_dir'],
                    os.path.join(self.config['pki_dir'], 'minions'),
                    os.path.join(self.config['pki_dir'], 'minions_pre'),
                    os.path.join(self.config['pki_dir'], 'minions_denied'),
                    os.path.join(self.config['pki_dir'],
                                 'minions_autosign'),
                    os.path.join(self.config['pki_dir'],
                                 'minions_rejected'),
                    self.config['cachedir'],
                    os.path.join(self.config['cachedir'], 'jobs'),
                    os.path.join(self.config['cachedir'], 'proc'),
                    self.config['sock_dir'],
                    self.config['token_dir'],
                    self.config['syndic_dir'],
                    self.config['sqlite_queue_dir'],
                ]
            if self.config.get('transport') == 'raet':
                v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted'))
                v_dirs.append(os.path.join(self.config['pki_dir'], 'pending'))
                v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected'))
                v_dirs.append(os.path.join(self.config['cachedir'], 'raet'))
            verify_env(
                v_dirs,
                self.config['user'],
                permissive=self.config['permissive_pki_access'],
                pki_dir=self.config['pki_dir'],
            )
            logfile = self.config['log_file']
            if logfile is not None and not logfile.startswith(('tcp://',
                                                               'udp://',
                                                               'file://')):
                # Logfile is not using Syslog, verify
                verify_files([logfile], self.config['user'])
            # Clear out syndics from cachedir
            for syndic_file in os.listdir(self.config['syndic_dir']):
                os.remove(os.path.join(self.config['syndic_dir'], syndic_file))
    except OSError as err:
        logger.exception('Failed to prepare salt environment')
        sys.exit(err.errno)

    self.setup_logfile_logger()
    logger.info('Setting up the Salt Master')

    # TODO: AIO core is separate from transport
    if self.config['transport'].lower() in ('zeromq', 'tcp'):
        if not verify_socket(self.config['interface'],
                             self.config['publish_port'],
                             self.config['ret_port']):
            self.exit(4, 'The ports are not available to bind\n')
        self.config['interface'] = ip_bracket(self.config['interface'])
        migrations.migrate_paths(self.config)

        # Late import so logging works correctly
        import salt.master
        self.master = salt.master.Master(self.config)
    else:
        # Add a udp port check here
        import salt.daemons.flo
        self.master = salt.daemons.flo.IofloMaster(self.config)
    self.daemonize_if_required()
    self.set_pidfile()
    salt.utils.process.notify_systemd()

def start(self):
    '''
    Start the actual master.

    If sub-classed, don't **ever** forget to run:

        super(YourSubClass, self).start()

    NOTE: Run any required code before calling `super()`.
    '''
    self.prepare() #调用自己的prepare方法
    if check_user(self.config['user']):
        logger.info('The salt master is starting up')
        self.master.start()

def shutdown(self):
    '''
    If sub-classed, run any shutdown operations on this method.
    '''
    logger.info('The salt master is shut down')

这里prepare salt variables, and environment, 然后调用salt.master的start方法, 类方法在: ~/salt/master.py

class Master(SMaster):
'''
The salt master server
'''
def __init__(self, opts):
    '''
    Create a salt master server instance

    :param dict: The salt options
    '''
    # Warn if ZMQ < 3.2
    try:
        zmq_version_info = zmq.zmq_version_info()
    except AttributeError:
        # PyZMQ <= 2.1.9 does not have zmq_version_info, fall back to
        # using zmq.zmq_version() and build a version info tuple.
        zmq_version_info = tuple(
            [int(x) for x in zmq.zmq_version().split('.')]
        )
    if zmq_version_info < (3, 2):
        log.warning(
            'You have a version of ZMQ less than ZMQ 3.2! There are '
            'known connection keep-alive issues with ZMQ < 3.2 which '
            'may result in loss of contact with minions. Please '
            'upgrade your ZMQ!'
        )
    SMaster.__init__(self, opts)

def start(self):
    '''
    Turn on the master server components
    '''
    self._pre_flight()
    log.info(
        'salt-master is starting as user {0!r}'.format(salt.utils.get_user())
    )

    enable_sigusr1_handler()
    enable_sigusr2_handler()

    self.__set_max_open_files()
    log.info('Creating master process manager')
    process_manager = salt.utils.process.ProcessManager()
    log.info('Creating master maintenance process')
    pub_channels = []
    for transport, opts in iter_transport_opts(self.opts):
        chan = salt.transport.server.PubServerChannel.factory(opts)
        chan.pre_fork(process_manager)
        pub_channels.append(chan)

    log.info('Creating master event publisher process')
    process_manager.add_process(salt.utils.event.EventPublisher, args=(self.opts,))
    salt.engines.start_engines(self.opts, process_manager)

    # must be after channels
    process_manager.add_process(Maintenance, args=(self.opts,))
    log.info('Creating master publisher process')

    if self.opts.get('reactor'):
        log.info('Creating master reactor process')
        process_manager.add_process(salt.utils.reactor.Reactor, args=(self.opts,))

    if self.opts.get('event_return'):
        log.info('Creating master event return process')
        process_manager.add_process(salt.utils.event.EventReturn, args=(self.opts,))

    ext_procs = self.opts.get('ext_processes', [])
    for proc in ext_procs:
        log.info('Creating ext_processes process: {0}'.format(proc))
        try:
        	mod = '.'.join(proc.split('.')[:-1])
            cls = proc.split('.')[-1]
            _tmp = __import__(mod, globals(), locals(), [cls], -1)
            cls = _tmp.__getattribute__(cls)
            process_manager.add_process(cls, args=(self.opts,))
        except Exception:
            log.error(('Error creating ext_processes '
                       'process: {0}').format(proc))

    if HAS_HALITE and 'halite' in self.opts:
        log.info('Creating master halite process')
        process_manager.add_process(Halite, args=(self.opts['halite'],))

    # TODO: remove, or at least push into the transport stuff (pre-fork probably makes sense there)
    if self.opts['con_cache']:
        log.info('Creating master concache process')
        process_manager.add_process(ConnectedCache, args=(self.opts,))
        # workaround for issue #16315, race condition
        log.debug('Sleeping for two seconds to let concache rest')
        time.sleep(2)

    log.info('Creating master request server process')
    process_manager.add_process(self.run_reqserver)
    try:
        process_manager.run()
    except KeyboardInterrupt:
        # Shut the master down gracefully on SIGINT
        log.warn('Stopping the Salt Master')
        process_manager.kill_children()
        raise SystemExit('\nExiting on Ctrl-c')

这里process_manager实例化的是process_manager = salt.utils.process.ProcessManager(), 使用add_process方法和run方法启动salt进程, 再看看ProcessMangeer, 目标类方法在: ~/salt/utils/process.py

class ProcessManager(object):
'''
A class which will manage processes that should be running
'''
def __init__(self, name=None, wait_for_kill=1):
    # pid -> {tgt: foo, Process: object, args: args, kwargs: kwargs}
    self._process_map = {}

    self.name = name
    if self.name is None:
        self.name = self.__class__.__name__

    self.wait_for_kill = wait_for_kill

    # store some pointers for the SIGTERM handler
    self._pid = os.getpid()
    self._sigterm_handler = signal.getsignal(signal.SIGTERM)

def add_process(self, tgt, args=None, kwargs=None):
    '''
    Create a processes and args + kwargs
    This will deterimine if it is a Process class, otherwise it assumes
    it is a function
    '''
    if args is None:
        args = []

    if kwargs is None:
        kwargs = {}

    if type(multiprocessing.Process) is type(tgt) and issubclass(tgt, multiprocessing.Process):
        process = tgt(*args, **kwargs)
    else:
        process = multiprocessing.Process(target=tgt, args=args, kwargs=kwargs)

    process.start()
    log.debug("Started '{0}' with pid {1}".format(tgt.__name__, process.pid))
    self._process_map[process.pid] = {'tgt': tgt,
                                      'args': args,
                                      'kwargs': kwargs,
                                      'Process': process}

def restart_process(self, pid):
    '''
    Create new process (assuming this one is dead), then remove the old one
    '''
    log.info('Process {0} ({1}) died with exit status {2},'
             ' restarting...'.format(self._process_map[pid]['tgt'],
                                     pid,
                                     self._process_map[pid]['Process'].exitcode))
    # don't block, the process is already dead
    self._process_map[pid]['Process'].join(1)

    self.add_process(self._process_map[pid]['tgt'],
                     self._process_map[pid]['args'],
                     self._process_map[pid]['kwargs'])

    del self._process_map[pid]

def run(self):
    '''
    Load and start all available api modules
    '''
    salt.utils.appendproctitle(self.name)

    # make sure to kill the subprocesses if the parent is killed
    signal.signal(signal.SIGTERM, self.kill_children)

    while True:
        try:
            # in case someone died while we were waiting...
            self.check_children()

            if not salt.utils.is_windows():
                pid, exit_status = os.wait()
                if pid not in self._process_map:
                    log.debug(('Process of pid {0} died, not a known'
                               ' process, will not restart').format(pid))
                    continue
                self.restart_process(pid)
            else:
                # os.wait() is not supported on Windows.
                time.sleep(10)
            # OSError is raised if a signal handler is called (SIGTERM) during os.wait
        except OSError:
            break

def check_children(self):
    '''
    Check the children once
    '''
    for pid, mapping in six.iteritems(self._process_map):
        if not mapping['Process'].is_alive():
            self.restart_process(pid)

def kill_children(self, *args):
    '''
    Kill all of the children
    '''
    # check that this is the correct process, children inherit this
    # handler, if we are in a child lets just run the original handler
    if os.getpid() != self._pid:
        if callable(self._sigterm_handler):
            return self._sigterm_handler(*args)
        elif self._sigterm_handler is not None:
            return signal.default_int_handler(signal.SIGTERM)(*args)
        else:
            return
    if salt.utils.is_windows():
        with open(os.devnull, 'wb') as devnull:
            for pid, p_map in six.iteritems(self._process_map):
                # On Windows, we need to explicitly terminate sub-processes
                # because the processes don't have a sigterm handler.
                subprocess.call(
                    ['taskkill', '/F', '/T', '/PID', str(pid)],
                    stdout=devnull, stderr=devnull
                    )
                p_map['Process'].terminate()
    else:
        for p_map in six.itervalues(self._process_map):
            p_map['Process'].terminate()

    end_time = time.time() + self.wait_for_kill  # when to die

    while self._process_map and time.time() < end_time:
        for pid, p_map in six.iteritems(self._process_map.copy()):
            p_map['Process'].join(0)

            # This is a race condition if a signal was passed to all children
            try:
                del self._process_map[pid]
            except KeyError:
                pass
    # if anyone is done after
    for pid in self._process_map:
        try:
            os.kill(signal.SIGKILL, pid)
        # in case the process has since decided to die, os.kill returns OSError
        except OSError:
            pass

add_process方法中实现是比较简单的, 调用Python的multiprocess库处理多线程.

---

最终使用了Python的MultiProcess库来完成多线程.

嗯, 先看看电视休息一下, 放松下凌乱的脑袋, 等会去脑补一下MultiProcess库 -