vswitchd: Integrate secchan into vswitchd.
authorBen Pfaff <blp@nicira.com>
Thu, 5 Mar 2009 20:30:36 +0000 (12:30 -0800)
committerBen Pfaff <blp@nicira.com>
Thu, 5 Mar 2009 20:30:36 +0000 (12:30 -0800)
This change means that vswitchd is now a single process that manages all
of the datapaths, instead of a parent process and any number of secchan
subprocesses.

It may be better to fork off processes that each manage a few bridges, to
take advantage of multiple cores.  Perhaps this is upcoming.

There are numerous new configuration file directives that are, sadly, not
yet documented.

vswitchd/automake.mk
vswitchd/bridge.c

index e6c68b8c17f7ebbb92d3042eff25c13bf1e82d81..6a263ca84d12c38978d63e09fc2e7aa795a41920 100644 (file)
@@ -13,6 +13,10 @@ vswitchd_vswitchd_SOURCES = \
        vswitchd/stats.c \
        vswitchd/stats.h \
        vswitchd/vswitchd.c
-vswitchd_vswitchd_LDADD = lib/libopenflow.a $(FAULT_LIBS) $(SSL_LIBS)
+vswitchd_vswitchd_LDADD = \
+       secchan/libsecchan.a \
+       lib/libopenflow.a \
+       $(FAULT_LIBS) \
+       $(SSL_LIBS)
 
 EXTRA_DIST += vswitchd/vswitchd.8.in
index 2554e339c38254dbe6968be7c75b2362bfa22139..aa70ad21013ec99d368ed6813f7521e7ca3b2ca4 100644 (file)
@@ -56,6 +56,7 @@
 #include "port-array.h"
 #include "process.h"
 #include "rconn.h"
+#include "secchan/ofproto.h"
 #include "socket-util.h"
 #include "stats.h"
 #include "stp.h"
@@ -63,6 +64,7 @@
 #include "timeval.h"
 #include "util.h"
 #include "vconn.h"
+#include "vconn-ssl.h"
 #include "xtoxll.h"
 
 #define THIS_MODULE VLM_bridge
@@ -156,18 +158,10 @@ struct bridge {
     char *controller;           /* NULL if there is no remote controller;
                                  * "discover" to do controller discovery;
                                  * otherwise a vconn name. */
-    struct svec secchan_opts;   /* Additional options for secchan. */
-
-    /* Secure channel. */
-    enum {
-        SC_UNSTARTED,           /* Not yet started. */
-        SC_RUNNING,             /* Started. */
-        SC_DYING,               /* 'rconn' is closed but process not dead. */
-        SC_DEAD                 /* Died too many times, giving up. */
-    } sc_state;
-    int sc_retries;             /* Number of times secchan restarted. */
-    struct process *secchan;    /* The "secchan" subprocess. */
-    struct rconn *rconn;        /* Connection to secchan subprocess. */
+
+    /* OpenFlow switch processing. */
+    struct ofproto *ofproto;    /* OpenFlow switch. */
+    struct rconn *rconn;        /* Connection to switch. */
 
     /* Kernel datapath information. */
     struct dpif dpif;           /* Kernel datapath. */
@@ -274,6 +268,42 @@ bridge_init(void)
     }
 }
 
+static bool
+config_string_change(const char *key, char **valuep)
+{
+    const char *value = cfg_get_string(0, "%s", key);
+    if (value && (!*valuep || strcmp(value, *valuep))) {
+        free(*valuep);
+        *valuep = xstrdup(value);
+        return true;
+    } else {
+        return false;
+    }
+}
+
+static void
+bridge_configure_ssl(void)
+{
+    /* XXX SSL should be configurable on a per-bridge basis.
+     * XXX should be possible to de-configure SSL. */
+    static char *private_key_file;
+    static char *certificate_file;
+    static char *cacert_file;
+
+    if (config_string_change("ssl.private-key", &private_key_file)) {
+        vconn_ssl_set_private_key_file(private_key_file);
+    }
+
+    if (config_string_change("ssl.certificate", &certificate_file)) {
+        vconn_ssl_set_certificate_file(certificate_file);
+    }
+
+    if (config_string_change("ssl.ca-cert", &cacert_file)) {
+        vconn_ssl_set_ca_cert_file(cacert_file,
+                                   cfg_get_bool(0, "ssl.bootstrap-ca-cert"));
+    }
+}
+
 void
 bridge_reconfigure(void)
 {
@@ -308,6 +338,9 @@ bridge_reconfigure(void)
     svec_destroy(&old_br);
     svec_destroy(&new_br);
 
+    /* Configure SSL. */
+    bridge_configure_ssl();
+
     /* Reconfigure all bridges. */
     LIST_FOR_EACH (br, struct bridge, node, &all_bridges) {
         bridge_reconfigure_one(br);
@@ -503,9 +536,7 @@ bridge_wait(void)
     struct bridge *br;
 
     LIST_FOR_EACH (br, struct bridge, node, &all_bridges) {
-        if (br->secchan) {
-            process_wait(br->secchan);
-        }
+        ofproto_wait(br->ofproto);
         if (br->controller) {
             continue;
         }
@@ -540,9 +571,6 @@ bridge_flush(struct bridge *br)
 \f
 /* Bridge reconfiguration functions. */
 
-static void run_secchan(struct bridge *);
-static void start_secchan(struct bridge *);
-
 static struct bridge *
 bridge_create(const char *name)
 {
@@ -551,6 +579,23 @@ bridge_create(const char *name)
 
     assert(!bridge_lookup(name));
     br = xcalloc(1, sizeof *br);
+
+    error = dpif_create(name, &br->dpif);
+    if (error) {
+        VLOG_ERR("failed to create datapath %s: %s", name, strerror(error));
+        free(br);
+        return NULL;
+    }
+
+    error = ofproto_create(name, &br->ofproto);
+    if (error) {
+        VLOG_ERR("failed to create OpenFlow switch %s: %s",
+                 name, strerror(error));
+        dpif_delete(&br->dpif);
+        dpif_close(&br->dpif);
+        return NULL;
+    }
+
     br->name = xstrdup(name);
     br->txqlen = rconn_packet_counter_create();
     br->ml = mac_learning_create();
@@ -559,11 +604,6 @@ bridge_create(const char *name)
     br->sent_features_request = false;
     eth_addr_random(br->default_ea);
 
-    svec_init(&br->secchan_opts);
-
-    br->sc_state = SC_UNSTARTED;
-    br->sc_retries = 0;
-    br->secchan = NULL;
     br->rconn = rconn_create(30, 1);
 
     port_array_init(&br->ifaces);
@@ -574,15 +614,6 @@ bridge_create(const char *name)
 
     br->stats_mgr = stats_mgr_create(br->rconn);
 
-    /* Create kernel datapath. */
-    error = dpif_create(br->name, &br->dpif);
-    if (error) {
-        VLOG_ERR("failed to create datapath %s: %s",
-                 br->name, strerror(error));
-        free(br);
-        return NULL;
-    }
-
     list_push_back(&all_bridges, &br->node);
 
     br->dp_idx = br->dpif.minor;
@@ -591,188 +622,37 @@ bridge_create(const char *name)
     return br;
 }
 
-
-static void
-log_secchan_died(enum vlog_level level, struct bridge *br, bool expected)
-{
-    char *status = process_status_msg(process_status(br->secchan));
-    vlog(THIS_MODULE, level, "%s: secchan subprocess with pid %ld died%s (%s)",
-         br->name, (long int) process_pid(br->secchan),
-         expected ? "" : " unexpectedly", status);
-    free(status);
-}
-
-static void
-hup_secchan(struct bridge *br)
-{
-    if (br->sc_state == SC_RUNNING) {
-        process_kill(br->secchan, SIGHUP);
-    }
-}
-
-static void
-kill_secchan(struct bridge *br)
-{
-    if (br->sc_state == SC_RUNNING) {
-        process_kill(br->secchan, SIGTERM);
-        br->sc_state = SC_DYING;
-    }
-}
-
-static void
-run_secchan(struct bridge *br)
-{
-    switch (br->sc_state) {
-    case SC_UNSTARTED:
-        start_secchan(br);
-        break;
-
-    case SC_RUNNING:
-        if (process_exited(br->secchan)) {
-            log_secchan_died(VLL_ERR, br, false);
-            br->sc_state = SC_UNSTARTED;
-        } else if (!br->controller && !rconn_is_alive(br->rconn)) {
-            VLOG_ERR("%s: connection to secchan closed unexpectedly, "
-                     "killing secchan", br->name);
-            kill_secchan(br);
-        }
-        break;
-
-    case SC_DYING:
-        if (process_exited(br->secchan)) {
-            log_secchan_died(VLL_WARN, br, true);
-            br->sc_state = SC_UNSTARTED;
-        }
-        break;
-
-    case SC_DEAD:
-        /* Nothing to do. */
-        break;
-    }
-}
-
-static bool
-console_logging_enabled(void)
-{
-    enum vlog_module module;
-
-    for (module = 0; module < VLM_N_MODULES; module++) {
-        if (vlog_get_level(module, VLF_CONSOLE) != VLL_EMER) {
-            return true;
-        }
-    }
-    return false;
-}
-
 static void
-start_secchan(struct bridge *br)
+connect_ofproto(struct bridge *br)
 {
-    int i;
-    struct svec argv;
+    static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
     int sockets[2];
-    struct stat s;
-    char *dp_name;
-    int retval;
+    int i;
 
-    /* Clean up vestiges of any previous secchan. */
-    rconn_disconnect(br->rconn);
-    if (br->secchan) {
-        process_destroy(br->secchan);
-        br->secchan = NULL;
+    if (ofproto_is_alive(br->ofproto)) {
+        return;
     }
 
-    /* Bail out if we've failed to start secchan too many times. */
-    if (br->sc_retries >= 5) {
-        VLOG_ERR("%s: restarted secchan maximum number of %d times, disabling",
-                 br->name, br->sc_retries);
-        br->sc_state = SC_DEAD;
+    VLOG_INFO_RL(&rl, "%s: connecting to internal OpenFlow switch", br->name);
+    if (socketpair(AF_LOCAL, SOCK_STREAM, 0, sockets)) {
+        VLOG_ERR("%s: failed to create socket pair: %s",
+                 br->name, strerror(errno));
+        /* XXX need to wait before trying again */
         return;
     }
-    br->sc_retries++;
-
-    if (!br->controller) {
-        /* Create socketpair for communicating with secchan subprocess. */
-        if (socketpair(AF_LOCAL, SOCK_STREAM, 0, sockets)) {
-            VLOG_ERR("%s: failed to create socket pair: %s",
-                     br->name, strerror(errno));
-            goto error;
-        }
-        set_nonblocking(sockets[0]);
-        set_nonblocking(sockets[1]);
 
-        /* Connect to our end of the socketpair. */
-        dp_name = xasprintf("fd:%d", sockets[0]);
-        rconn_connect(br->rconn, dp_name);
-        free(dp_name);
-    } else {
-        /* secchan will connect to the external controller, not to us, so there
-         * is nothing for us to do. */
-    }
-
-    /* Assemble command-line arguments. */
-    svec_init(&argv);
-    svec_add(&argv, "secchan");
-    svec_add_nocopy(&argv, xasprintf("--br-name=%s", br->name));
-    for (i=0; i<config_files.n; i++) {
-        svec_add_nocopy(&argv, xasprintf("--config=%s", 
-                    config_files.names[i]));
-    }
-    svec_add_nocopy(&argv, xasprintf("-vPATTERN:console:%s|secchan-%s|%s",
-                                     "%d{%b %d %H:%M:%S}", br->name,
-                                     "%c|%p|%m"));
-    svec_add_nocopy(&argv, xasprintf("-vPATTERN:syslog:secchan-%s|%s",
-                                     br->name, "%05N|%c|%p|%m"));
-    if (!console_logging_enabled()) {
-        svec_add(&argv, "-vANY:console:EMER");
-    }
-    if (vlog_get_log_file()) {
-        svec_add_nocopy(&argv, xasprintf("--log-file=%s.secchan-%s",
-                                         vlog_get_log_file(), br->name));
-    }
-    if (!br->controller) {
-        svec_add(&argv, "--out-of-band");
-        svec_add(&argv, "--max-backoff=1");
-    }
-    svec_add(&argv, "--fail=closed");
-    svec_add(&argv, "--no-stp");
-    if (!stat(ofp_rundir, &s)) {
-        svec_add_nocopy(&argv,
-                        xasprintf("--listen=punix:%s/secchan.mgmt-%s",
-                                  ofp_rundir, br->name));
-    }
-    svec_append(&argv, &br->secchan_opts);
-    svec_add_nocopy(&argv, xasprintf("nl:%d", br->dp_idx));
-    if (br->controller) {
-        if (strcmp(br->controller, "discover")) {
-            svec_add(&argv, br->controller);
+    /* Connect the ofproto to one end of the socketpair, and ourselves to the
+     * other end. */
+    for (i = 0; i < 2; i++) {
+        char *name = xasprintf("fd:%d", sockets[i]);
+        set_nonblocking(sockets[i]);
+        if (i) {
+            ofproto_set_controller(br->ofproto, name);
+        } else {
+            rconn_connect(br->rconn, name);
         }
-    } else {
-        svec_add_nocopy(&argv, xasprintf("fd:%d", sockets[1]));
+        free(name);
     }
-    svec_terminate(&argv);
-
-    /* Start secchan. */
-    if (!br->controller) {
-        retval = process_start(argv.names, &sockets[1], 1, NULL, 0,
-                               &br->secchan);
-        close(sockets[1]);
-    } else {
-        retval = process_start(argv.names, NULL, 0, NULL, 0, &br->secchan);
-    }
-    svec_destroy(&argv);
-    if (retval) {
-        VLOG_ERR("%s: failed to start secchan for datapath nl:%d: %s",
-                 br->name, br->dp_idx, strerror(retval));
-        goto error;
-    }
-    br->sc_state = SC_RUNNING;
-    br->sent_config_request = false;
-    br->sent_features_request = false;
-    br->next_stats_request = time_now();
-    return;
-
-error:
-    br->sc_state = SC_UNSTARTED;
 }
 
 static void
@@ -790,14 +670,10 @@ bridge_destroy(struct bridge *br)
                          br->dp_idx, strerror(retval));
             }
         }
-        if (br->secchan) {
-            process_kill(br->secchan, SIGTERM);
-        }
-        process_destroy(br->secchan);
+        ofproto_destroy(br->ofproto);
         rconn_destroy(br->rconn);
         rconn_packet_counter_destroy(br->txqlen);
         free(br->controller);
-        svec_destroy(&br->secchan_opts);
         ft_destroy(br->ft);
         stats_request_destroy(br->flow_rq);
         stats_mgr_destroy(br->stats_mgr);
@@ -874,7 +750,7 @@ bridge_run_one(struct bridge *br)
     int iteration;
 
     if (br->controller) {
-        run_secchan(br);
+        ofproto_run(br->ofproto);
         return;
     }
 
@@ -908,8 +784,9 @@ bridge_run_one(struct bridge *br)
     bond_run(br);
     brstp_run(br);
 
-    /* Start or restart secchan if necessary. */
-    run_secchan(br);
+    /* Start or restart switch if necessary. */
+    connect_ofproto(br);
+    ofproto_run(br->ofproto);
 
     /* Now revalidate any flows that need it. */
     if (br->flush || !tag_set_is_empty(&br->revalidate_set)) {
@@ -930,9 +807,10 @@ static void
 bridge_reconfigure_one(struct bridge *br)
 {
     struct svec old_ports, new_ports, ifaces;
+    struct svec listeners, old_listeners;
     const char *controller;
     size_t i, j;
-    char *ctl;
+    char *ctl, *pfx;
 
     /* Collect old and new ports. */
     svec_init(&old_ports);
@@ -1000,39 +878,130 @@ bridge_reconfigure_one(struct bridge *br)
     svec_destroy(&ifaces);
 
     /* Configure remote controller. */
-    controller = cfg_get_string(0, "bridge.%s.controller", br->name);
+    pfx = xasprintf("bridge.%s.controller", br->name);
+    controller = cfg_get_string(0, "%s", pfx);
     ctl = controller ? xstrdup(controller) : NULL;
-    if ((ctl == NULL) != (br->controller == NULL)
-        || (ctl && br->controller && strcmp(ctl, br->controller))) {
-        br->sc_retries = 0;
-        kill_secchan(br);
+    if (ctl) {
+        const char *fail_mode;
+
+        if (!strcmp(ctl, "discover")) {
+            ofproto_set_discovery(br->ofproto, true,
+                                  cfg_get_string(0,
+                                                 "%s.accept-controller-regex",
+                                                 pfx),
+                                  cfg_get_bool(0, "%s.update-resolv.conf",
+                                               pfx));
+        } else {
+            struct netdev *netdev;
+            bool in_band;
+            int error;
+
+            in_band = (!cfg_is_valid(CFG_BOOL | CFG_REQUIRED,
+                                     "%s.in-band", pfx)
+                       || cfg_get_bool(0, "%s.in-band", pfx));
+            ofproto_set_discovery(br->ofproto, false, NULL, NULL);
+            ofproto_set_in_band(br->ofproto, in_band);
+            if (!br->controller || !strcmp(ctl, br->controller)) {
+                ofproto_set_controller(br->ofproto, ctl);
+            }
+
+            error = netdev_open(br->name, NETDEV_ETH_TYPE_NONE, &netdev);
+            if (!error) {
+                netdev_turn_flags_on(netdev, NETDEV_UP, true);
+                if (cfg_is_valid(CFG_IP | CFG_REQUIRED, "%s.ip", pfx)) {
+                    struct in_addr ip, mask, gateway;
+                    ip.s_addr = cfg_get_ip(0, "%s.ip", pfx);
+                    mask.s_addr = cfg_get_ip(0, "%s.netmask", pfx);
+                    gateway.s_addr = cfg_get_ip(0, "%s.gateway", pfx);
+
+                    if (!mask.s_addr) {
+                        mask.s_addr = guess_netmask(ip.s_addr);
+                    }
+                    if (!netdev_set_in4(netdev, ip, mask)) {
+                        VLOG_INFO("bridge %s: configured IP address "IP_FMT", "
+                                  "netmask "IP_FMT,
+                                  br->name, IP_ARGS(&ip.s_addr),
+                                  IP_ARGS(&mask.s_addr));
+                    }
+
+                    if (gateway.s_addr) {
+                        if (!netdev_add_router(gateway)) {
+                            VLOG_INFO("bridge %s: configured gateway "IP_FMT,
+                                      br->name, IP_ARGS(&gateway.s_addr));
+                        }
+                    }
+                }
+                netdev_close(netdev);
+            }
+        }
+
+        fail_mode = cfg_get_string(0, "%s.fail-mode", pfx);
+        ofproto_set_failure(br->ofproto,
+                            fail_mode && (!strcmp(fail_mode, "standalone") ||
+                                          !strcmp(fail_mode, "open")));
+        ofproto_set_probe_interval(br->ofproto,
+                                   cfg_get_int(0, "%s.inactivity-probe", pfx));
+        ofproto_set_max_backoff(br->ofproto,
+                                cfg_get_int(0, "%s.max-backoff", pfx));
+        ofproto_set_stp(br->ofproto, cfg_get_bool(0, "%s.stp", pfx));
+
+        if (cfg_has("%s.commands.acl", pfx)) {
+            struct svec command_acls;
+            char *command_acl;
+
+            svec_init(&command_acls);
+            cfg_get_all_strings(&command_acls, "%s.commands.acl", pfx);
+            command_acl = svec_join(&command_acls, ",");
+
+            ofproto_set_remote_execution(br->ofproto, command_acl,
+                                         cfg_get_string(0, "%s.commands.dir",
+                                                        pfx));
+
+            svec_destroy(&command_acls);
+            free(command_acl);
+        } else {
+            ofproto_set_remote_execution(br->ofproto, NULL, NULL);
+        }
+    } else {
+        if (br->controller) {
+            /* There was a controller configured, so we want to disconnect
+             * from it so that connect_ofproto() will set up a "loopback"
+             * connection to us over a socketpair. */
+            ofproto_set_controller(br->ofproto, NULL);
+        } else {
+            /* No controller configured before either, so we're already doing
+             * the right thing. */
+        }
+        ofproto_set_in_band(br->ofproto, false);
+        ofproto_set_max_backoff(br->ofproto, 1);
+        ofproto_set_probe_interval(br->ofproto, 5);
+        ofproto_set_failure(br->ofproto, false);
+        ofproto_set_stp(br->ofproto, false);
     }
     free(br->controller);
     br->controller = ctl;
+    free(pfx);
 
-    /* Allow arbitrary secchan options if a remote controller is configured. */
-    svec_clear(&br->secchan_opts);
-    if (ctl) {
-        for (i = 0; ; i++) {
-            const char *opt;
-            opt = cfg_get_string(i, "bridge.%s.secchan.options", br->name);
-            if (!opt) {
-                break;
-            }
-            svec_parse_words(&br->secchan_opts, opt);
-        }
+    /* Configure OpenFlow management listeners. */
+    svec_init(&listeners);
+    cfg_get_all_strings(&listeners, "bridge.%s.openflow.listeners", br->name);
+    if (!listeners.n) {
+        svec_add_nocopy(&listeners, xasprintf("punix:%s/secchan.mgmt-%s",
+                                              ofp_rundir, br->name));
     }
+    svec_sort_unique(&listeners);
+
+    svec_init(&old_listeners);
+    ofproto_get_listeners(br->ofproto, &old_listeners);
+    svec_sort_unique(&old_listeners);
 
-    /* Revive secchan if it's dead. */
-    if (br->sc_state == SC_DEAD) {
-        br->sc_retries = 0;
-        br->sc_state = SC_UNSTARTED;
+    if (!svec_equal(&listeners, &old_listeners)) {
+        ofproto_set_listeners(br->ofproto, &listeners);
     }
+    svec_destroy(&listeners);
+    svec_destroy(&old_listeners);
 
     mirror_reconfigure(br);
-
-    /* Force secchan to reconfigure itself. */
-    hup_secchan(br);
 }
 
 static void
@@ -1177,8 +1146,8 @@ queue_tx(struct bridge *br, struct ofpbuf *msg)
     if (retval) {
         ofpbuf_delete(msg);
         /* No point in logging: rconn_send() only fails due to disconnection,
-         * and disconnect from secchan will cause all kinds of log messages
-         * anyhow.  */
+         * and disconnecting from the switch will cause all kinds of log
+         * messages anyhow.  */
     }
 }