systemd bug: bz1437114 core:execute: fix fork() fail handling in exec_spawn()

问题现象

大量僵尸进程

root     32278  0.0  0.0      0     0 ?        Z    05:39   0:00 [runuser] <defunct>
root     32280  0.0  0.0      0     0 ?        Z    Aug22   0:00 [runuser] <defunct>
root     32304  0.0  0.0      0     0 ?        Z    Aug21   0:00 [runuser] <defunct>
root     32362  0.0  0.0      0     0 ?        Z    Aug22   0:00 [runuser] <defunct>
root     32448  0.0  0.0      0     0 ?        Z    Aug22   0:00 [runuser] <defunct>
root     32475  0.0  0.0      0     0 ?        Z    Aug20   0:00 [runuser] <defunct>
root     32498  0.0  0.0      0     0 ?        Z    Aug22   0:00 [runuser] <defunct>
root     32509  0.0  0.0      0     0 ?        Z    Aug18   0:00 [runuser] <defunct>
root     32551  0.0  0.0      0     0 ?        Z    09:54   0:00 [runuser] <defunct>
root     32556  0.0  0.0      0     0 ?        Z    Aug18   0:00 [runuser] <defunct>
root     32557  0.0  0.0      0     0 ?        Z    Aug18   0:00 [runuser] <defunct>
root     32611  0.0  0.0      0     0 ?        Z    Aug22   0:00 [runuser] <defunct>
root     32612  0.0  0.0      0     0 ?        Z    Aug22   0:00 [runuser] <defunct>
root     32647  0.0  0.0      0     0 ?        Z    Aug19   0:00 [runuser] <defunct>
root     32699  0.0  0.0      0     0 ?        Z    Aug18   0:00 [runuser] <defunct>
root     32700  0.0  0.0      0     0 ?        Z    Aug18   0:00 [runuser] <defunct>
root     32716  0.0  0.0      0     0 ?        Z    Aug19   0:00 [runuser] <defunct>
root     32753  0.0  0.0      0     0 ?        Z    Aug18   0:00 [runuser] <defunct>

systemd crash

#pstack 1
#0  0x00007fba436a4a2d in pause () from /lib64/libpthread.so.0
#1  0x00007fba44c66ecd in freeze ()
#2  0x00007fba44c14eef in crash.2986 ()
#3  <signal handler called>
#4  0x00007fba433095f7 in raise () from /lib64/libc.so.6
#5  0x00007fba4330ace8 in abort () from /lib64/libc.so.6
#6  0x00007fba44c85c72 in log_assert_failed ()
#7  0x00007fba44c4635d in unit_watch_pid ()
#8  0x00007fba44be3767 in service_spawn.9373 ()
#9  0x00007fba44c5ecab in service_start.9527 ()
#10 0x00007fba44bdc1ba in manager_dispatch_run_queue.3886 ()
#11 0x00007fba44c37db2 in source_dispatch.52334.3984 ()
#12 0x00007fba44c3adba in sd_event_dispatch ()
#13 0x00007fba44c0c9ef in manager_loop ()
#14 0x00007fba44bcb86c in main ()

(gdb)
(gdb) bt
#0  0x00007fba436a4a2d in pause () from /lib64/libpthread.so.0
#1  0x00007fba44c66ecd in freeze () at src/shared/util.c:3833
#2  0x00007fba44c14eef in crash.2986 (sig=<optimized out>) at src/core/main.c:223
#3  <signal handler called>
#4  0x00007fba433095f7 in raise () from /lib64/libc.so.6
#5  0x00007fba4330ace8 in abort () from /lib64/libc.so.6
#6  0x00007fba44c85c72 in log_assert_failed (text=<optimized out>, file=<optimized out>, line=<optimized out>, func=<optimized out>) at src/shared/log.c:753
#7  0x00007fba44c4635d in unit_watch_pid (u=0x7fba44e44250, pid=0) at src/core/unit.c:2017
#8  0x00007fba44be3767 in service_spawn.9373 (s=0x7fba44e44250, c=0x7fba68f07ad0, timeout=90000000, pass_fds=<optimized out>, apply_permissions=<optimized out>, apply_chroot=<optimized out>,
    apply_tty_stdin=true, is_control=true, _pid=0x7fba44e4486c) at src/core/service.c:1170
#9  0x00007fba44c5ecab in service_enter_start_pre (s=0x7fba44e44250) at src/core/service.c:1596
#10 service_start.9527 (u=0x7fba44e44250) at src/core/service.c:1841
#11 0x00007fba44bdc1ba in manager_dispatch_run_queue.3886 (source=<optimized out>, userdata=0x7fba44d132e0) at src/core/unit.c:1481
#12 0x00007fba44c37db2 in source_dispatch (s=s@entry=0x7fba44d11ea0) at src/libsystemd/sd-event/sd-event.c:2155
#13 0x00007fba44c3adba in sd_event_dispatch (e=0x7fba44d11c30) at src/libsystemd/sd-event/sd-event.c:2472
#14 0x00007fba44c0c9ef in sd_event_run (timeout=18446744073709551615, e=0x7fba44d11c30) at src/libsystemd/sd-event/sd-event.c:2501
#15 manager_loop (m=0x7fba44d132e0) at src/core/manager.c:2070
#16 0x00007fba44bcb86c in main (argc=5, argv=0x7ffc45ee3a38) at src/core/main.c:1755
(gdb)

问题代码定位

(gdb) list unit_watch_pid
2008
2009	        unit_add_to_dbus_queue(u);
2010	        unit_add_to_gc_queue(u);
2011	}
2012
2013	int unit_watch_pid(Unit *u, pid_t pid) {
2014	        int q, r;
2015
2016	        assert(u);
2017	        assert(pid >= 1);   # 这里触发
(gdb)

(gdb) list src/core/service.c:1170
                return r;
static int service_load_pid_file(Service *s, bool may_warn) {

1165	                       s->exec_runtime,
1166	                       &pid);
1167	        if (r < 0)
1168	                goto fail;
1169
1170	        r = unit_watch_pid(UNIT(s), pid);  # 这里调用
1171	        if (r < 0)
1172	                /* FIXME: we need to do something here */
1173	                goto fail;
1174

(gdb) list src/core/service.c:1596
	static void service_enter_start_pre(Service *s) {

1591	                 * be left from previous runs. */
1592	                service_kill_control_processes(s);
1593
1594	                s->control_command_id = SERVICE_EXEC_START_PRE;
1595
1596	                r = service_spawn(s,
1597	                                  s->control_command,
1598	                                  s->timeout_start_usec,
1599	                                  false,
1600	                                  !s->permissions_start_only

复现方法

1. 创建一个vm中测试
2. 想办法耗尽内存

# mount -t tmpfs -o size=20G tmpfs /mnt
# dd if=/dev/zero of=/mnt/file bs=1M

1. while :; do systemctl restart iptables.service; sleep 5; done

分析

问题发生在当系统内存耗尽，恰好在此时，有个systemd的service重启，此时，systemd需要分配一个pid，但是因为内存耗尽，无法分配pid。然后触发了systemd一个bug：

BUG存在于: 创建pid函数exec_spawn(), 当创建pid失败的时候，理应返回错误值，但是这里返回的是0。 这个0是r决定的。 这个r是r = exec_context_load_environment() 函数的返回值，然而这个函数永远都是返回成功，导致，pid < 0 pid创建失败后，依然返回0

创建pid函数exec_spawn():
int exec_spawn
        pid = fork();
        if (pid < 0)
                return log_unit_error_errno(params->unit_id, r, "Failed to fork: %m");

service重启，systemd会为service创建一个pid，在pid<0 的情况下，执行到unit_watch_pid() 函数，unit_watch_pid()函数判断assert(pid >= 1);

service_spawn()函数-> 调用exec_spawn()
重启service 会调用service_spawn()函数

static int service_spawn(...
        r = exec_spawn(c,
                       &s->exec_context,
                       &exec_params,
                       s->exec_runtime,
                       &pid);
        if (r < 0)
                goto fail;

        r = unit_watch_pid(UNIT(s), pid);
        if (r < 0)
                /* FIXME: we need to do something here */
                goto fail;

        *_pid = pid;

int unit_watch_pid(Unit *u, pid_t pid) {
        int q, r;

        assert(u);
        assert(pid >= 1);

REF

patch:

https://github.com/lnykryn/systemd-rhel/pull/119/commits/e18a5fa348aae0800807c3121a1ccf750eca206f

https://github.com/systemd/systemd/pull/5886

solution:

https://access.redhat.com/solutions/3096191