From: Ben Pfaff Date: Fri, 15 Jan 2010 20:13:46 +0000 (-0800) Subject: daemon: Add support for process monitoring and restart. X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ff8decf1a318b4a611cb08bb3f12833044e8a872;p=openvswitch daemon: Add support for process monitoring and restart. --- diff --git a/lib/daemon.c b/lib/daemon.c index 9895f725..86291146 100644 --- a/lib/daemon.c +++ b/lib/daemon.c @@ -25,6 +25,7 @@ #include "fatal-signal.h" #include "dirs.h" #include "lockfile.h" +#include "process.h" #include "socket-util.h" #include "timeval.h" #include "util.h" @@ -47,6 +48,10 @@ static bool chdir_ = true; /* File descriptor used by daemonize_start() and daemonize_complete(). */ static int daemonize_fd = -1; +/* --monitor: Should a supervisory process monitor the daemon and restart it if + * it dies due to an error signal? */ +static bool monitor; + /* Returns the file name that would be used for a pidfile if 'name' were * provided to set_pidfile(). The caller must free the returned string. */ char * @@ -117,6 +122,14 @@ get_detach(void) return detach; } +/* Sets up a following call to daemonize() to fork a supervisory process to + * monitor the daemon and restart it if it dies due to an error signal. */ +void +daemon_set_monitor(void) +{ + monitor = true; +} + /* If a pidfile has been configured and that pidfile already exists and is * locked by a running process, returns the pid of the running process. * Otherwise, returns 0. */ @@ -286,6 +299,69 @@ fork_notify_startup(int fd) } } +static bool +should_restart(int status) +{ + if (WIFSIGNALED(status)) { + static const int error_signals[] = { + SIGABRT, SIGALRM, SIGBUS, SIGFPE, SIGILL, SIGPIPE, SIGSEGV, + SIGXCPU, SIGXFSZ + }; + + size_t i; + + for (i = 0; i < ARRAY_SIZE(error_signals); i++) { + if (error_signals[i] == WTERMSIG(status)) { + return true; + } + } + } + return false; +} + +static void +monitor_daemon(pid_t daemon_pid) +{ + /* XXX Should limit the rate at which we restart the daemon. */ + /* XXX Should log daemon's stderr output at startup time. */ + const char *saved_program_name; + + saved_program_name = program_name; + program_name = xasprintf("monitor(%s)", program_name); + for (;;) { + int retval; + int status; + + do { + retval = waitpid(daemon_pid, &status, 0); + } while (retval == -1 && errno == EINTR); + + if (retval == -1) { + ovs_fatal(errno, "waitpid failed"); + } else if (retval == daemon_pid) { + char *status_msg = process_status_msg(status); + if (should_restart(status)) { + VLOG_ERR("%s daemon died unexpectedly (%s), restarting", + saved_program_name, status_msg); + free(status_msg); + + daemon_pid = fork_and_wait_for_startup(&daemonize_fd); + if (!daemon_pid) { + break; + } + } else { + VLOG_INFO("%s daemon exited normally (%s), exiting", + saved_program_name, status_msg); + exit(0); + } + } + } + + /* Running in new daemon process. */ + free((char *) program_name); + program_name = saved_program_name; +} + /* Close stdin, stdout, stderr. If we're started from e.g. an SSH session, * then this keeps us from holding that session open artificially. */ static void @@ -314,6 +390,20 @@ daemonize_start(void) /* Running in parent process. */ exit(0); } + /* Running in daemon or monitor process. */ + } + + if (monitor) { + int saved_daemonize_fd = daemonize_fd; + pid_t daemon_pid; + + daemon_pid = fork_and_wait_for_startup(&daemonize_fd); + if (daemon_pid > 0) { + /* Running in monitor process. */ + fork_notify_startup(saved_daemonize_fd); + close_standard_fds(); + monitor_daemon(daemon_pid); + } /* Running in daemon process. */ } diff --git a/lib/daemon.h b/lib/daemon.h index 8ded63f1..1d630768 100644 --- a/lib/daemon.h +++ b/lib/daemon.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2009 Nicira Networks. + * Copyright (c) 2008, 2009, 2010 Nicira Networks. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,13 +26,15 @@ enum { OPT_NO_CHDIR, OPT_OVERWRITE_PIDFILE, OPT_PIDFILE, + OPT_MONITOR }; #define DAEMON_LONG_OPTIONS \ {"detach", no_argument, 0, OPT_DETACH}, \ {"no-chdir", no_argument, 0, OPT_NO_CHDIR}, \ {"pidfile", optional_argument, 0, OPT_PIDFILE}, \ - {"overwrite-pidfile", no_argument, 0, OPT_OVERWRITE_PIDFILE} + {"overwrite-pidfile", no_argument, 0, OPT_OVERWRITE_PIDFILE},\ + {"monitor", no_argument, 0, OPT_MONITOR} #define DAEMON_OPTION_HANDLERS \ case OPT_DETACH: \ @@ -49,6 +51,10 @@ enum { \ case OPT_OVERWRITE_PIDFILE: \ ignore_existing_pidfile(); \ + break; \ + \ + case OPT_MONITOR: \ + daemon_set_monitor(); \ break; char *make_pidfile_name(const char *name); @@ -58,6 +64,7 @@ void set_no_chdir(void); bool is_chdir_enabled(void); void set_detach(void); bool get_detach(void); +void daemon_set_monitor(void); void daemonize(void); void daemonize_start(void); void daemonize_complete(void); diff --git a/lib/daemon.man b/lib/daemon.man index ea0561da..c2393774 100644 --- a/lib/daemon.man +++ b/lib/daemon.man @@ -19,6 +19,17 @@ When \fB--pidfile\fR is not specified, this option has no effect. Causes \fB\*(PN\fR to detach itself from the foreground session and run as a background process. +.TP +\fB--monitor\fR +Creates an additional process to monitor the \fB\*(PN\fR daemon. If +the daemon dies due to a signal that indicates a programming error +(e.g. \fBSIGSEGV\fR, \fBSIGABRT\fR), then the monitor process starts a +new copy of it. If the daemon die or exits for another reason, the +monitor process exits. +.IP +This option is normally used with \fB--detach\fR, but it also +functions without it. + .TP \fB--no-chdir\fR By default, when \fB--detach\fR is specified, \fB\*(PN\fR diff --git a/tests/automake.mk b/tests/automake.mk index 7a73a84f..dc677eb1 100644 --- a/tests/automake.mk +++ b/tests/automake.mk @@ -9,6 +9,7 @@ TESTSUITE_AT = \ tests/ovsdb-macros.at \ tests/lcov-pre.at \ tests/library.at \ + tests/daemon.at \ tests/vconn.at \ tests/dir_name.at \ tests/aes128.at \ diff --git a/tests/daemon.at b/tests/daemon.at new file mode 100644 index 00000000..1062229a --- /dev/null +++ b/tests/daemon.at @@ -0,0 +1,165 @@ +AT_BANNER([daemon unit tests]) + +AT_SETUP([daemon]) +OVSDB_INIT([db]) +AT_CAPTURE_FILE([pid]) +AT_CAPTURE_FILE([expected]) +# Start the daemon and wait for the pidfile to get created +# and that its contents are the correct pid. +AT_CHECK([ovsdb-server --pidfile=$PWD/pid --remote=punix:socket --unixctl=$PWD/unixctl db& echo $! > expected], [0], [ignore], [ignore]) +OVS_WAIT_UNTIL([test -s pid], [kill `cat expected`]) +AT_CHECK( + [pid=`cat pid` && expected=`cat expected` && test "$pid" = "$expected"], + [0], [], [], [kill `cat expected`]) +AT_CHECK([kill -0 `cat pid`], [0], [], [], [kill `cat expected`]) +# Kill the daemon and make sure that the pidfile gets deleted. +kill `cat expected` +OVS_WAIT_WHILE([kill -0 `cat expected`]) +AT_CHECK([test ! -e pid]) +AT_CLEANUP + +AT_SETUP([daemon --monitor]) +OVSDB_INIT([db]) +AT_CAPTURE_FILE([pid]) +AT_CAPTURE_FILE([parent]) +AT_CAPTURE_FILE([parentpid]) +AT_CAPTURE_FILE([newpid]) +# Start the daemon and wait for the pidfile to get created. +AT_CHECK([ovsdb-server --monitor --pidfile=$PWD/pid --remote=punix:socket --unixctl=$PWD/unixctl db& echo $! > parent], [0], [ignore], [ignore]) +OVS_WAIT_UNTIL([test -s pid], [kill `cat parent`]) +# Check that the pidfile names a running process, +# and that the parent process of that process is our child process. +AT_CHECK([kill -0 `cat pid`], [0], [], [], [kill `cat parent`]) +AT_CHECK([ps -o ppid= -p `cat pid` > parentpid], + [0], [], [], [kill `cat parent`]) +AT_CHECK( + [parentpid=`cat parentpid` && + parent=`cat parent` && + test $parentpid = $parent], + [0], [], [], [kill `cat parent`]) +# Kill the daemon process, making it look like a segfault, +# and wait for a new child process to get spawned. +AT_CHECK([cp pid oldpid], [0], [], [], [kill `cat parent`]) +AT_CHECK([kill -SEGV `cat pid`], [0], [], [ignore], [kill `cat parent`]) +OVS_WAIT_WHILE([kill -0 `cat oldpid`], [kill `cat parent`]) +OVS_WAIT_UNTIL([test -s pid && test `cat pid` != `cat oldpid`], + [kill `cat parent`]) +AT_CHECK([cp pid newpid], [0], [], [], [kill `cat parent`]) +# Check that the pidfile names a running process, +# and that the parent process of that process is our child process. +AT_CHECK([ps -o ppid= -p `cat pid` > parentpid], + [0], [], [], [kill `cat parent`]) +AT_CHECK( + [parentpid=`cat parentpid` && + parent=`cat parent` && + test $parentpid = $parent], + [0], [], [], [kill `cat parent`]) +# Kill the daemon process with SIGTERM, and wait for the daemon +# and the monitor processes to go away and the pidfile to get deleted. +AT_CHECK([kill `cat pid`], [0], [], [ignore], [kill `cat parent`]) +OVS_WAIT_WHILE([kill -0 `cat parent` || kill -0 `cat newpid` || test -e pid], + [kill `cat parent`]) +AT_CLEANUP + +AT_SETUP([daemon --detach]) +AT_CAPTURE_FILE([pid]) +OVSDB_INIT([db]) +# Start the daemon and make sure that the pidfile exists immediately. +# We don't wait for the pidfile to get created because the daemon is +# supposed to do so before the parent exits. +AT_CHECK([ovsdb-server --detach --pidfile=$PWD/pid --remote=punix:socket --unixctl=$PWD/unixctl db], [0], [ignore], [ignore]) +AT_CHECK([test -s pid]) +AT_CHECK([kill -0 `cat pid`]) +# Kill the daemon and make sure that the pidfile gets deleted. +cp pid saved-pid +kill `cat pid` +OVS_WAIT_WHILE([kill -0 `cat saved-pid`]) +AT_CHECK([test ! -e pid]) +AT_CLEANUP + +AT_SETUP([daemon --detach --monitor]) +m4_define([CHECK], + [AT_CHECK([$1], [$2], [$3], [$4], [kill `cat daemon monitor`])]) +OVSDB_INIT([db]) +AT_CAPTURE_FILE([daemon]) +AT_CAPTURE_FILE([olddaemon]) +AT_CAPTURE_FILE([newdaemon]) +AT_CAPTURE_FILE([monitor]) +AT_CAPTURE_FILE([newmonitor]) +AT_CAPTURE_FILE([init]) +# Start the daemon and make sure that the pidfile exists immediately. +# We don't wait for the pidfile to get created because the daemon is +# supposed to do so before the parent exits. +AT_CHECK([ovsdb-server --detach --pidfile=$PWD/daemon --monitor --remote=punix:socket --unixctl=$PWD/unixctl db], [0], [ignore], [ignore]) +AT_CHECK([test -s daemon]) +# Check that the pidfile names a running process, +# and that the parent process of that process is a running process, +# and that the parent process of that process is init. +CHECK([kill -0 `cat daemon`]) +CHECK([ps -o ppid= -p `cat daemon` > monitor]) +CHECK([kill -0 `cat monitor`]) +CHECK([ps -o ppid= -p `cat monitor` > init]) +CHECK([test `cat init` = 1]) +# Kill the daemon process, making it look like a segfault, +# and wait for a new daemon process to get spawned. +CHECK([cp daemon olddaemon]) +CHECK([kill -SEGV `cat daemon`], [0], [ignore], [ignore]) +OVS_WAIT_WHILE([kill -0 `cat olddaemon`], [kill `cat olddaemon daemon`]) +OVS_WAIT_UNTIL([test -s daemon && test `cat daemon` != `cat olddaemon`], + [kill `cat olddaemon daemon`]) +CHECK([cp daemon newdaemon]) +# Check that the pidfile names a running process, +# and that the parent process of that process is our child process. +CHECK([kill -0 `cat daemon`]) +CHECK([diff olddaemon newdaemon], [1], [ignore]) +CHECK([ps -o ppid= -p `cat daemon` > newmonitor]) +CHECK([diff monitor newmonitor]) +CHECK([kill -0 `cat newmonitor`]) +CHECK([ps -o ppid= -p `cat newmonitor` > init]) +CHECK([test `cat init` = 1]) +# Kill the daemon process with SIGTERM, and wait for the daemon +# and the monitor processes to go away and the pidfile to get deleted. +CHECK([kill `cat daemon`], [0], [], [ignore]) +OVS_WAIT_WHILE( + [kill -0 `cat monitor` || kill -0 `cat newdaemon` || test -e daemon], + [kill `cat monitor newdaemon`]) +m4_undefine([CHECK]) +AT_CLEANUP + +AT_SETUP([daemon --detach startup errors]) +AT_CAPTURE_FILE([pid]) +OVSDB_INIT([db]) +OVS_CHECK_LCOV([ovsdb-server --detach --pidfile=$PWD/pid --unixctl=$PWD/nonexistent/unixctl db], [1], [], [stderr]) +AT_CHECK([grep 'ovsdb-server: could not initialize control socket' stderr], + [0], [ignore], []) +AT_CHECK([test ! -s pid]) +AT_CLEANUP + +AT_SETUP([daemon --detach --monitor startup errors]) +AT_CAPTURE_FILE([pid]) +OVSDB_INIT([db]) +OVS_CHECK_LCOV([ovsdb-server --detach --pidfile=$PWD/pid --monitor --unixctl=$PWD/nonexistent/unixctl db], [1], [], [stderr]) +AT_CHECK([grep 'ovsdb-server: could not initialize control socket' stderr], + [0], [ignore], []) +AT_CHECK([test ! -s pid]) +AT_CLEANUP + +AT_SETUP([daemon --detach closes standard fds]) +AT_CAPTURE_FILE([pid]) +OVSDB_INIT([db]) +AT_CHECK([(yes; echo $? > status) | ovsdb-server --detach --pidfile=$PWD/pid --unixctl=$PWD/unixctl db], [0], [], [stderr]) +AT_CHECK([kill `cat pid`]) +AT_CHECK([test -s status]) +AT_CHECK([kill -l `cat status`], [0], [PIPE +]) +AT_CLEANUP + +AT_SETUP([daemon --detach --monitor closes standard fds]) +AT_CAPTURE_FILE([pid]) +OVSDB_INIT([db]) +AT_CHECK([(yes; echo $? > status) | ovsdb-server --detach --monitor --pidfile=$PWD/pid --unixctl=$PWD/unixctl db], [0], [], [stderr]) +AT_CHECK([kill `cat pid`]) +AT_CHECK([test -s status]) +AT_CHECK([kill -l `cat status`], [0], [PIPE +]) +AT_CLEANUP diff --git a/tests/testsuite.at b/tests/testsuite.at index 6ec1a809..93d7e6eb 100644 --- a/tests/testsuite.at +++ b/tests/testsuite.at @@ -39,6 +39,7 @@ m4_include([tests/ovsdb-macros.at]) m4_include([tests/lcov-pre.at]) m4_include([tests/library.at]) +m4_include([tests/daemon.at]) m4_include([tests/vconn.at]) m4_include([tests/dir_name.at]) m4_include([tests/aes128.at])