xenserver: allow dom0 traffic in secure pool host when controller unavailable.
authorDavid Tsai <dtsai@nicira.com>
Fri, 17 Jun 2011 06:13:24 +0000 (23:13 -0700)
committerBen Pfaff <blp@nicira.com>
Fri, 17 Jun 2011 17:54:38 +0000 (10:54 -0700)
A pool configured for secure fail-mode can block dom0 traffic on hosts joining
the pool or if the host reboots while the controller is unavailable.  This
commit sets default flows on a host under these conditions to allow management
traffic.  Once the connection with the controller is re-established, these
default flows are replaced by the controller.

tests/interface-reconfigure.at updated by Ben Pfaff.

NIC-376.

Signed-off-by: David Tsai <dtsai@nicira.com>
Signed-off-by: Ben Pfaff <blp@nicira.com>
tests/interface-reconfigure.at
xenserver/etc_xapi.d_plugins_openvswitch-cfg-update
xenserver/opt_xensource_libexec_InterfaceReconfigureVswitch.py

index b9871f3e1616cbcdffac133ce6dad77b6b159e43..90ca127c98d0b4ff282a7723f9fb2f6bc56dcb37 100644 (file)
@@ -40,7 +40,6 @@ EOF
         sbin/ip \
         sbin/update-issue \
         sbin/vconfig \
-        usr/bin/ovs-vsctl \
         usr/sbin/brctl \
         usr/sbin/ovs-vlan-bug-workaround
     do
@@ -52,6 +51,36 @@ EOF
         chmod +x $utility
     done
 
+    mkdir -p usr/bin
+    cat > usr/bin/ovs-vsctl <<'EOF'
+#! /bin/sh
+echo ${0} ${*} >&2
+
+while test ${#} -ge 4; do
+    if test X"${1}" = Xget && \
+       test X"${2}" = Xinterface && \
+       test X"${4}" = Xofport; then
+          if test X"${3}" = Xeth2; then
+             echo 5
+         else
+             echo -1
+         fi
+    fi
+
+    shift
+done
+EOF
+    chmod +x usr/bin/ovs-vsctl
+
+    cat > usr/bin/ovs-ofctl <<'EOF'
+#! /bin/sh
+echo ${0} ${*} >&2
+
+# Check that the flow is properly formed.
+ovs-ofctl parse-flow "${3}" >/dev/null
+EOF
+    chmod +x usr/bin/ovs-ofctl
+
     mkdir -p etc/sysconfig/network-scripts
     configure_netdev () {
         mkdir -p sys/class/net/${1}
@@ -644,7 +673,7 @@ EOF
         <pool ref="OpaqueRef:a765d06c-fc82-cc67-8f6c-fd8db45f6a84">
                 <other_config>
                         <vswitch-controller-fail-mode>
-                                standalone
+                                secure
                         </vswitch-controller-fail-mode>
                 </other_config>
         </pool>
@@ -674,6 +703,7 @@ configure_datapath: bridge      - xenbr2
 configure_datapath: physical    - [u'eth2']
 configure_datapath: extra ports - []
 configure_datapath: extra bonds - []
+/usr/bin/ovs-vsctl --timeout=5 -vANY:console:emer get-fail-mode xenbr2
 Applying changes to /etc/sysconfig/network-scripts/route-xenbr2 configuration
 Applying changes to /etc/sysconfig/network configuration
 Applying changes to /etc/sysconfig/network-scripts/ifcfg-xenbr2 configuration
@@ -685,9 +715,14 @@ Applying changes to /etc/sysconfig/network-scripts/ifcfg-xenbr2 configuration
     --may-exist add-br xenbr2
     --may-exist add-port xenbr2 eth2
     set Bridge xenbr2 other-config:hwaddr="00:15:17:a0:29:80"
-    set Bridge xenbr2 fail_mode=standalone
+    set Bridge xenbr2 fail_mode=secure
     remove Bridge xenbr2 other_config disable-in-band
     br-set-external-id xenbr2 xs-network-uuids d08c8749-0c8f-9e8d-ce25-fd364661ee99
+/usr/bin/ovs-vsctl --timeout=5 -vANY:console:emer get interface eth2 ofport
+/usr/bin/ovs-ofctl add-flow xenbr2 idle_timeout=0,priority=0,in_port=5,arp,nw_proto=1,actions=local
+/usr/bin/ovs-ofctl add-flow xenbr2 idle_timeout=0,priority=0,in_port=local,arp,dl_src=00:15:17:a0:29:80,actions=5
+/usr/bin/ovs-ofctl add-flow xenbr2 idle_timeout=0,priority=0,in_port=5,dl_dst=00:15:17:a0:29:80,actions=local
+/usr/bin/ovs-ofctl add-flow xenbr2 idle_timeout=0,priority=0,in_port=local,dl_src=00:15:17:a0:29:80,actions=5
 /sbin/ifup xenbr2
 /sbin/update-issue
 Committing changes to /etc/sysconfig/network-scripts/route-xenbr2 configuration
@@ -752,7 +787,7 @@ Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi3 configuration
     --may-exist add-br xenbr3
     --may-exist add-port xenbr3 eth3
     set Bridge xenbr3 other-config:hwaddr="00:15:17:a0:29:81"
-    set Bridge xenbr3 fail_mode=standalone
+    set Bridge xenbr3 fail_mode=secure
     remove Bridge xenbr3 other_config disable-in-band
     br-set-external-id xenbr3 xs-network-uuids 2902ae1b-8013-897a-b697-0b200ea3aaa5;db7bdc03-074d-42ae-fc73-9b06de1d57f6
     --if-exists del-br xapi3
@@ -843,7 +878,7 @@ Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi1 configuration
     --fake-iface add-bond xapi1 bond0 eth0 eth1
     set Port bond0 MAC="00:22:19:22:4b:af" other-config:bond-miimon-interval=100 bond_downdelay=200 bond_updelay=31000 other-config:bond-detect-mode=carrier lacp=off bond_mode=balance-slb
     set Bridge xapi1 other-config:hwaddr="00:22:19:22:4b:af"
-    set Bridge xapi1 fail_mode=standalone
+    set Bridge xapi1 fail_mode=secure
     remove Bridge xapi1 other_config disable-in-band
     br-set-external-id xapi1 xs-network-uuids 45cbbb43-113d-a712-3231-c6463f253cef;99be2da4-6c33-6f8e-49ea-3bc592fe3c85
 /sbin/ifup xapi1
@@ -930,7 +965,7 @@ Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi2 configuration
     --fake-iface add-bond xapi1 bond0 eth0 eth1
     set Port bond0 MAC="00:22:19:22:4b:af" other-config:bond-miimon-interval=100 bond_downdelay=200 bond_updelay=31000 other-config:bond-detect-mode=carrier lacp=off bond_mode=balance-slb
     set Bridge xapi1 other-config:hwaddr="00:22:19:22:4b:af"
-    set Bridge xapi1 fail_mode=standalone
+    set Bridge xapi1 fail_mode=secure
     remove Bridge xapi1 other_config disable-in-band
     br-set-external-id xapi1 xs-network-uuids 45cbbb43-113d-a712-3231-c6463f253cef;99be2da4-6c33-6f8e-49ea-3bc592fe3c85
     --if-exists del-br xapi2
index f001d2f61706cdac56863870cfb48a6b960de582..bceccbf4cb985f3c6aa31e8fb73c5de354fd617b 100755 (executable)
@@ -26,8 +26,10 @@ import XenAPI
 import os
 import subprocess
 import syslog
+import re
 
 vsctl="/usr/bin/ovs-vsctl"
+ofctl="/usr/bin/ovs-ofctl"
 cacert_filename="/etc/openvswitch/vswitchd.cacert"
 
 # Delete the CA certificate, so that we go back to boot-strapping mode
@@ -54,6 +56,7 @@ def update(session, args):
         raise XenAPIPlugin.Failure("NO_POOL_FOR_HOST", [])
     if len(pools) > 1:
         raise XenAPIPlugin.Failure("MORE_THAN_ONE_POOL_FOR_HOST", [])
+    new_controller = False
     pool = session.xenapi.pool.get_record(pools[0])
     controller = pool.get("vswitch_controller", "")
     ret_str = ""
@@ -73,6 +76,7 @@ def update(session, args):
         except:
             pass
         setControllerCfg(controller)
+        new_controller = True
         ret_str += "Successfully set controller to %s.  " % controller
 
     try:
@@ -89,6 +93,18 @@ def update(session, args):
         except KeyError:
             pass
 
+    # If new controller, get managagment MAC addresses from XAPI now
+    # in case fail_mode set to secure which may affect XAPI access
+    mgmt_bridge = None
+    host_mgmt_mac = None
+    host_mgmt_device = None
+    pool_mgmt_macs = {}
+    if new_controller:
+        for n in session.xenapi.PIF.get_all():
+            rec = session.xenapi.PIF.get_record(n)
+            if rec.get('management', False):
+                pool_mgmt_macs[rec.get('MAC')] = rec.get('device')
+
     dib_changed = False
     fail_mode_changed = False
     for bridge in vswitchCfgQuery(['list-br']).split():
@@ -141,6 +157,29 @@ def update(session, args):
                 "fail_mode=%s" % fail_mode])
             fail_mode_changed = True
 
+        # Determine local mgmt MAC address if host being added to secure
+        # pool so we can add default flows to allow management traffic
+        if new_controller and fail_mode_changed and pool_fail_mode == "secure":
+            oc = vswitchCfgQuery(["get", "Bridge", bridge, "other-config"])
+            m = re.match('.*hwaddr="([0-9a-fA-F:].*)".*', oc)
+            if m and m.group(1) in pool_mgmt_macs.keys():
+                mgmt_bridge = bridge
+                host_mgmt_mac = m.group(1)
+                host_mgmt_device = pool_mgmt_macs[host_mgmt_mac]
+
+    if host_mgmt_mac is not None and mgmt_bridge is not None and \
+        host_mgmt_device is not None:
+        tp = "idle_timeout=0,priority=0"
+        port = vswitchCfgQuery(["get", "interface", host_mgmt_device, "ofport"])
+        addFlow(mgmt_bridge, "%s,in_port=%s,arp,nw_proto=1,actions=local" % \
+            (tp, port))
+        addFlow(mgmt_bridge, "%s,in_port=local,arp,dl_src=%s,actions=%s" % \
+            (tp, host_mgmt_mac, port))
+        addFlow(mgmt_bridge, "%s,in_port=%s,dl_dst=%s,actions=local" % \
+            (tp, port, host_mgmt_mac))
+        addFlow(mgmt_bridge, "%s,in_port=local,dl_src=%s,actions=%s" % \
+            (tp, host_mgmt_mac, port))
+
     if dib_changed:
         ret_str += "Updated in-band management.  "
     if fail_mode_changed:
@@ -199,6 +238,13 @@ def emergency_reset(session, args):
                                    [ str(exitcode) ])
 
     return "Successfully reset configuration"
+
+def addFlow(switch, flow):
+    cmd = [ofctl, "add-flow", switch, flow]
+    exitcode = subprocess.call(cmd)
+    if exitcode != 0:
+        raise XenAPIPlugin.Failure("VSWITCH_ADD_FLOW_FAILURE",
+                                   [ str(exitcode) , str(switch), str(flow) ])
     
 if __name__ == "__main__":
     XenAPIPlugin.dispatch({"update": update,
index 10c6bd20f7442791255f6f0cd8baf1069a8b68c9..c223e41391c58ee811d5ce6a7a03423083b3da58 100644 (file)
@@ -14,6 +14,7 @@
 from InterfaceReconfigure import *
 import os
 import re
+import subprocess
 
 #
 # Bare Network Devices -- network devices without IP configuration
@@ -292,10 +293,13 @@ def configure_datapath(pif):
     - A list containing the necessary vsctl command line arguments
     - A list of additional devices which should be brought up after
       the configuration is applied.
+    - A list containing flows to apply to the pif bridge, note that
+      port numbers may need to be substituted once ofport is known
     """
 
     vsctl_argv = []
     extra_up_ports = []
+    bridge_flows = []
 
     assert not pif_is_vlan(pif)
     bridge = pif_bridge_name(pif)
@@ -404,6 +408,25 @@ def configure_datapath(pif):
 
     if (fail_mode not in valid_fail_modes) and pool:
         fail_mode = pool['other_config'].get('vswitch-controller-fail-mode')
+        # Add default flows to allow management traffic if fail-mode
+        # transitions to secure based on pool fail-mode setting
+        if fail_mode == 'secure' and db().get_pif_record(pif).get('management', False):
+            prev_fail_mode = vswitchCfgQuery(['get-fail-mode', bridge])
+            if prev_fail_mode != 'secure':
+                tp = 'idle_timeout=0,priority=0'
+                host_mgmt_mac = db().get_pif_record(pif)['MAC']
+                # account for bond as management interface
+                if len(physical_devices) > 1:
+                    bridge_flows += ['%s,in_port=local,arp,dl_src=%s,actions=NORMAL' % (tp, host_mgmt_mac)]
+                    bridge_flows += ['%s,in_port=local,dl_src=%s,actions=NORMAL' % (tp, host_mgmt_mac)]
+                    # we don't know slave ofports yet, substitute later
+                    bridge_flows += ['%s,in_port=%%s,arp,nw_proto=1,actions=local' % (tp)]
+                    bridge_flows += ['%s,in_port=%%s,dl_dst=%s,actions=local' % (tp, host_mgmt_mac)]
+                else:
+                    bridge_flows += ['%s,in_port=%%s,arp,nw_proto=1,actions=local' % (tp)]
+                    bridge_flows += ['%s,in_port=local,arp,dl_src=%s,actions=%%s' % (tp, host_mgmt_mac)]
+                    bridge_flows += ['%s,in_port=%%s,dl_dst=%s,actions=local' % (tp, host_mgmt_mac)]
+                    bridge_flows += ['%s,in_port=local,dl_src=%s,actions=%%s' % (tp, host_mgmt_mac)]
 
     if fail_mode not in valid_fail_modes:
         fail_mode = 'standalone'
@@ -422,7 +445,7 @@ def configure_datapath(pif):
     vsctl_argv += set_br_external_ids(pif)
     vsctl_argv += ['## done configuring datapath %s' % bridge]
 
-    return vsctl_argv,extra_up_ports
+    return vsctl_argv,extra_up_ports,bridge_flows
 
 def deconfigure_bridge(pif):
     vsctl_argv = []
@@ -475,6 +498,7 @@ class DatapathVswitch(Datapath):
         Datapath.__init__(self, pif)
         self._dp = pif_datapath(pif)
         self._ipdev = pif_ipdev_name(pif)
+        self._bridge_flows = []
 
         if pif_is_vlan(pif) and not self._dp:
             raise Error("Unbridged VLAN devices not implemented yet")
@@ -505,15 +529,17 @@ class DatapathVswitch(Datapath):
     def preconfigure(self, parent):
         vsctl_argv = []
         extra_ports = []
+        bridge_flows = []
 
         pifrec = db().get_pif_record(self._pif)
         dprec = db().get_pif_record(self._dp)
 
         ipdev = self._ipdev
-        c,e = configure_datapath(self._dp)
+        c,e,f = configure_datapath(self._dp)
         bridge = pif_bridge_name(self._pif)
         vsctl_argv += c
         extra_ports += e
+        bridge_flows += f
 
         dpname = pif_bridge_name(self._dp)
         
@@ -542,6 +568,7 @@ class DatapathVswitch(Datapath):
 
         self._vsctl_argv = vsctl_argv
         self._extra_ports = extra_ports
+        self._bridge_flows = bridge_flows
 
     def bring_down_existing(self):
         # interface-reconfigure is never explicitly called to down a
@@ -612,6 +639,26 @@ class DatapathVswitch(Datapath):
             run_command(['/usr/sbin/ovs-vlan-bug-workaround', dev, setting])
 
         datapath_modify_config(self._vsctl_argv)
+        if self._bridge_flows:
+            ofports = []
+            physical_devices = datapath_get_physical_pifs(self._dp)
+            if len(physical_devices) > 1:
+                for slave in physical_devices:
+                    name = pif_netdev_name(slave)
+                    ofport = vswitchCfgQuery(['get', 'interface', name, 'ofport'])
+                    ofports.append(ofport)
+            else:
+                name = pif_netdev_name(self._dp)
+                ofport = vswitchCfgQuery(['get', 'interface', name, 'ofport'])
+                ofports.append(ofport)
+            dpname = pif_bridge_name(self._dp)
+            for flow in self._bridge_flows:
+                if flow.find('in_port=%s') != -1 or flow.find('actions=%s') != -1:
+                    for port in ofports:
+                        f = flow % (port)
+                        run_command(['/usr/bin/ovs-ofctl', 'add-flow', dpname, f])
+                else:
+                    run_command(['/usr/bin/ovs-ofctl', 'add-flow', dpname, flow])
 
     def post(self):
         for p in self._extra_ports:
@@ -667,3 +714,17 @@ class DatapathVswitch(Datapath):
                 netdev_down(p)
 
         datapath_modify_config(vsctl_argv)
+
+#
+# utility methods
+#
+
+def vswitchCfgQuery(action_args):
+    cmd = ['%s/usr/bin/ovs-vsctl' % root_prefix(),
+        '--timeout=5', '-vANY:console:emer'] + action_args
+    output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()
+    if len(output) == 0 or output[0] == None:
+        output = ""
+    else:
+        output = output[0].strip()
+    return output