From: David Tsai Date: Fri, 17 Jun 2011 06:13:24 +0000 (-0700) Subject: xenserver: allow dom0 traffic in secure pool host when controller unavailable. X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=da7198b411d68df604db1a6132cf3be5c21842db;p=openvswitch xenserver: allow dom0 traffic in secure pool host when controller unavailable. A pool configured for secure fail-mode can block dom0 traffic on hosts joining the pool or if the host reboots while the controller is unavailable. This commit sets default flows on a host under these conditions to allow management traffic. Once the connection with the controller is re-established, these default flows are replaced by the controller. tests/interface-reconfigure.at updated by Ben Pfaff. NIC-376. Signed-off-by: David Tsai Signed-off-by: Ben Pfaff --- diff --git a/tests/interface-reconfigure.at b/tests/interface-reconfigure.at index b9871f3e..90ca127c 100644 --- a/tests/interface-reconfigure.at +++ b/tests/interface-reconfigure.at @@ -40,7 +40,6 @@ EOF sbin/ip \ sbin/update-issue \ sbin/vconfig \ - usr/bin/ovs-vsctl \ usr/sbin/brctl \ usr/sbin/ovs-vlan-bug-workaround do @@ -52,6 +51,36 @@ EOF chmod +x $utility done + mkdir -p usr/bin + cat > usr/bin/ovs-vsctl <<'EOF' +#! /bin/sh +echo ${0} ${*} >&2 + +while test ${#} -ge 4; do + if test X"${1}" = Xget && \ + test X"${2}" = Xinterface && \ + test X"${4}" = Xofport; then + if test X"${3}" = Xeth2; then + echo 5 + else + echo -1 + fi + fi + + shift +done +EOF + chmod +x usr/bin/ovs-vsctl + + cat > usr/bin/ovs-ofctl <<'EOF' +#! /bin/sh +echo ${0} ${*} >&2 + +# Check that the flow is properly formed. +ovs-ofctl parse-flow "${3}" >/dev/null +EOF + chmod +x usr/bin/ovs-ofctl + mkdir -p etc/sysconfig/network-scripts configure_netdev () { mkdir -p sys/class/net/${1} @@ -644,7 +673,7 @@ EOF - standalone + secure @@ -674,6 +703,7 @@ configure_datapath: bridge - xenbr2 configure_datapath: physical - [u'eth2'] configure_datapath: extra ports - [] configure_datapath: extra bonds - [] +/usr/bin/ovs-vsctl --timeout=5 -vANY:console:emer get-fail-mode xenbr2 Applying changes to /etc/sysconfig/network-scripts/route-xenbr2 configuration Applying changes to /etc/sysconfig/network configuration Applying changes to /etc/sysconfig/network-scripts/ifcfg-xenbr2 configuration @@ -685,9 +715,14 @@ Applying changes to /etc/sysconfig/network-scripts/ifcfg-xenbr2 configuration --may-exist add-br xenbr2 --may-exist add-port xenbr2 eth2 set Bridge xenbr2 other-config:hwaddr="00:15:17:a0:29:80" - set Bridge xenbr2 fail_mode=standalone + set Bridge xenbr2 fail_mode=secure remove Bridge xenbr2 other_config disable-in-band br-set-external-id xenbr2 xs-network-uuids d08c8749-0c8f-9e8d-ce25-fd364661ee99 +/usr/bin/ovs-vsctl --timeout=5 -vANY:console:emer get interface eth2 ofport +/usr/bin/ovs-ofctl add-flow xenbr2 idle_timeout=0,priority=0,in_port=5,arp,nw_proto=1,actions=local +/usr/bin/ovs-ofctl add-flow xenbr2 idle_timeout=0,priority=0,in_port=local,arp,dl_src=00:15:17:a0:29:80,actions=5 +/usr/bin/ovs-ofctl add-flow xenbr2 idle_timeout=0,priority=0,in_port=5,dl_dst=00:15:17:a0:29:80,actions=local +/usr/bin/ovs-ofctl add-flow xenbr2 idle_timeout=0,priority=0,in_port=local,dl_src=00:15:17:a0:29:80,actions=5 /sbin/ifup xenbr2 /sbin/update-issue Committing changes to /etc/sysconfig/network-scripts/route-xenbr2 configuration @@ -752,7 +787,7 @@ Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi3 configuration --may-exist add-br xenbr3 --may-exist add-port xenbr3 eth3 set Bridge xenbr3 other-config:hwaddr="00:15:17:a0:29:81" - set Bridge xenbr3 fail_mode=standalone + set Bridge xenbr3 fail_mode=secure remove Bridge xenbr3 other_config disable-in-band br-set-external-id xenbr3 xs-network-uuids 2902ae1b-8013-897a-b697-0b200ea3aaa5;db7bdc03-074d-42ae-fc73-9b06de1d57f6 --if-exists del-br xapi3 @@ -843,7 +878,7 @@ Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi1 configuration --fake-iface add-bond xapi1 bond0 eth0 eth1 set Port bond0 MAC="00:22:19:22:4b:af" other-config:bond-miimon-interval=100 bond_downdelay=200 bond_updelay=31000 other-config:bond-detect-mode=carrier lacp=off bond_mode=balance-slb set Bridge xapi1 other-config:hwaddr="00:22:19:22:4b:af" - set Bridge xapi1 fail_mode=standalone + set Bridge xapi1 fail_mode=secure remove Bridge xapi1 other_config disable-in-band br-set-external-id xapi1 xs-network-uuids 45cbbb43-113d-a712-3231-c6463f253cef;99be2da4-6c33-6f8e-49ea-3bc592fe3c85 /sbin/ifup xapi1 @@ -930,7 +965,7 @@ Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi2 configuration --fake-iface add-bond xapi1 bond0 eth0 eth1 set Port bond0 MAC="00:22:19:22:4b:af" other-config:bond-miimon-interval=100 bond_downdelay=200 bond_updelay=31000 other-config:bond-detect-mode=carrier lacp=off bond_mode=balance-slb set Bridge xapi1 other-config:hwaddr="00:22:19:22:4b:af" - set Bridge xapi1 fail_mode=standalone + set Bridge xapi1 fail_mode=secure remove Bridge xapi1 other_config disable-in-band br-set-external-id xapi1 xs-network-uuids 45cbbb43-113d-a712-3231-c6463f253cef;99be2da4-6c33-6f8e-49ea-3bc592fe3c85 --if-exists del-br xapi2 diff --git a/xenserver/etc_xapi.d_plugins_openvswitch-cfg-update b/xenserver/etc_xapi.d_plugins_openvswitch-cfg-update index f001d2f6..bceccbf4 100755 --- a/xenserver/etc_xapi.d_plugins_openvswitch-cfg-update +++ b/xenserver/etc_xapi.d_plugins_openvswitch-cfg-update @@ -26,8 +26,10 @@ import XenAPI import os import subprocess import syslog +import re vsctl="/usr/bin/ovs-vsctl" +ofctl="/usr/bin/ovs-ofctl" cacert_filename="/etc/openvswitch/vswitchd.cacert" # Delete the CA certificate, so that we go back to boot-strapping mode @@ -54,6 +56,7 @@ def update(session, args): raise XenAPIPlugin.Failure("NO_POOL_FOR_HOST", []) if len(pools) > 1: raise XenAPIPlugin.Failure("MORE_THAN_ONE_POOL_FOR_HOST", []) + new_controller = False pool = session.xenapi.pool.get_record(pools[0]) controller = pool.get("vswitch_controller", "") ret_str = "" @@ -73,6 +76,7 @@ def update(session, args): except: pass setControllerCfg(controller) + new_controller = True ret_str += "Successfully set controller to %s. " % controller try: @@ -89,6 +93,18 @@ def update(session, args): except KeyError: pass + # If new controller, get managagment MAC addresses from XAPI now + # in case fail_mode set to secure which may affect XAPI access + mgmt_bridge = None + host_mgmt_mac = None + host_mgmt_device = None + pool_mgmt_macs = {} + if new_controller: + for n in session.xenapi.PIF.get_all(): + rec = session.xenapi.PIF.get_record(n) + if rec.get('management', False): + pool_mgmt_macs[rec.get('MAC')] = rec.get('device') + dib_changed = False fail_mode_changed = False for bridge in vswitchCfgQuery(['list-br']).split(): @@ -141,6 +157,29 @@ def update(session, args): "fail_mode=%s" % fail_mode]) fail_mode_changed = True + # Determine local mgmt MAC address if host being added to secure + # pool so we can add default flows to allow management traffic + if new_controller and fail_mode_changed and pool_fail_mode == "secure": + oc = vswitchCfgQuery(["get", "Bridge", bridge, "other-config"]) + m = re.match('.*hwaddr="([0-9a-fA-F:].*)".*', oc) + if m and m.group(1) in pool_mgmt_macs.keys(): + mgmt_bridge = bridge + host_mgmt_mac = m.group(1) + host_mgmt_device = pool_mgmt_macs[host_mgmt_mac] + + if host_mgmt_mac is not None and mgmt_bridge is not None and \ + host_mgmt_device is not None: + tp = "idle_timeout=0,priority=0" + port = vswitchCfgQuery(["get", "interface", host_mgmt_device, "ofport"]) + addFlow(mgmt_bridge, "%s,in_port=%s,arp,nw_proto=1,actions=local" % \ + (tp, port)) + addFlow(mgmt_bridge, "%s,in_port=local,arp,dl_src=%s,actions=%s" % \ + (tp, host_mgmt_mac, port)) + addFlow(mgmt_bridge, "%s,in_port=%s,dl_dst=%s,actions=local" % \ + (tp, port, host_mgmt_mac)) + addFlow(mgmt_bridge, "%s,in_port=local,dl_src=%s,actions=%s" % \ + (tp, host_mgmt_mac, port)) + if dib_changed: ret_str += "Updated in-band management. " if fail_mode_changed: @@ -199,6 +238,13 @@ def emergency_reset(session, args): [ str(exitcode) ]) return "Successfully reset configuration" + +def addFlow(switch, flow): + cmd = [ofctl, "add-flow", switch, flow] + exitcode = subprocess.call(cmd) + if exitcode != 0: + raise XenAPIPlugin.Failure("VSWITCH_ADD_FLOW_FAILURE", + [ str(exitcode) , str(switch), str(flow) ]) if __name__ == "__main__": XenAPIPlugin.dispatch({"update": update, diff --git a/xenserver/opt_xensource_libexec_InterfaceReconfigureVswitch.py b/xenserver/opt_xensource_libexec_InterfaceReconfigureVswitch.py index 10c6bd20..c223e413 100644 --- a/xenserver/opt_xensource_libexec_InterfaceReconfigureVswitch.py +++ b/xenserver/opt_xensource_libexec_InterfaceReconfigureVswitch.py @@ -14,6 +14,7 @@ from InterfaceReconfigure import * import os import re +import subprocess # # Bare Network Devices -- network devices without IP configuration @@ -292,10 +293,13 @@ def configure_datapath(pif): - A list containing the necessary vsctl command line arguments - A list of additional devices which should be brought up after the configuration is applied. + - A list containing flows to apply to the pif bridge, note that + port numbers may need to be substituted once ofport is known """ vsctl_argv = [] extra_up_ports = [] + bridge_flows = [] assert not pif_is_vlan(pif) bridge = pif_bridge_name(pif) @@ -404,6 +408,25 @@ def configure_datapath(pif): if (fail_mode not in valid_fail_modes) and pool: fail_mode = pool['other_config'].get('vswitch-controller-fail-mode') + # Add default flows to allow management traffic if fail-mode + # transitions to secure based on pool fail-mode setting + if fail_mode == 'secure' and db().get_pif_record(pif).get('management', False): + prev_fail_mode = vswitchCfgQuery(['get-fail-mode', bridge]) + if prev_fail_mode != 'secure': + tp = 'idle_timeout=0,priority=0' + host_mgmt_mac = db().get_pif_record(pif)['MAC'] + # account for bond as management interface + if len(physical_devices) > 1: + bridge_flows += ['%s,in_port=local,arp,dl_src=%s,actions=NORMAL' % (tp, host_mgmt_mac)] + bridge_flows += ['%s,in_port=local,dl_src=%s,actions=NORMAL' % (tp, host_mgmt_mac)] + # we don't know slave ofports yet, substitute later + bridge_flows += ['%s,in_port=%%s,arp,nw_proto=1,actions=local' % (tp)] + bridge_flows += ['%s,in_port=%%s,dl_dst=%s,actions=local' % (tp, host_mgmt_mac)] + else: + bridge_flows += ['%s,in_port=%%s,arp,nw_proto=1,actions=local' % (tp)] + bridge_flows += ['%s,in_port=local,arp,dl_src=%s,actions=%%s' % (tp, host_mgmt_mac)] + bridge_flows += ['%s,in_port=%%s,dl_dst=%s,actions=local' % (tp, host_mgmt_mac)] + bridge_flows += ['%s,in_port=local,dl_src=%s,actions=%%s' % (tp, host_mgmt_mac)] if fail_mode not in valid_fail_modes: fail_mode = 'standalone' @@ -422,7 +445,7 @@ def configure_datapath(pif): vsctl_argv += set_br_external_ids(pif) vsctl_argv += ['## done configuring datapath %s' % bridge] - return vsctl_argv,extra_up_ports + return vsctl_argv,extra_up_ports,bridge_flows def deconfigure_bridge(pif): vsctl_argv = [] @@ -475,6 +498,7 @@ class DatapathVswitch(Datapath): Datapath.__init__(self, pif) self._dp = pif_datapath(pif) self._ipdev = pif_ipdev_name(pif) + self._bridge_flows = [] if pif_is_vlan(pif) and not self._dp: raise Error("Unbridged VLAN devices not implemented yet") @@ -505,15 +529,17 @@ class DatapathVswitch(Datapath): def preconfigure(self, parent): vsctl_argv = [] extra_ports = [] + bridge_flows = [] pifrec = db().get_pif_record(self._pif) dprec = db().get_pif_record(self._dp) ipdev = self._ipdev - c,e = configure_datapath(self._dp) + c,e,f = configure_datapath(self._dp) bridge = pif_bridge_name(self._pif) vsctl_argv += c extra_ports += e + bridge_flows += f dpname = pif_bridge_name(self._dp) @@ -542,6 +568,7 @@ class DatapathVswitch(Datapath): self._vsctl_argv = vsctl_argv self._extra_ports = extra_ports + self._bridge_flows = bridge_flows def bring_down_existing(self): # interface-reconfigure is never explicitly called to down a @@ -612,6 +639,26 @@ class DatapathVswitch(Datapath): run_command(['/usr/sbin/ovs-vlan-bug-workaround', dev, setting]) datapath_modify_config(self._vsctl_argv) + if self._bridge_flows: + ofports = [] + physical_devices = datapath_get_physical_pifs(self._dp) + if len(physical_devices) > 1: + for slave in physical_devices: + name = pif_netdev_name(slave) + ofport = vswitchCfgQuery(['get', 'interface', name, 'ofport']) + ofports.append(ofport) + else: + name = pif_netdev_name(self._dp) + ofport = vswitchCfgQuery(['get', 'interface', name, 'ofport']) + ofports.append(ofport) + dpname = pif_bridge_name(self._dp) + for flow in self._bridge_flows: + if flow.find('in_port=%s') != -1 or flow.find('actions=%s') != -1: + for port in ofports: + f = flow % (port) + run_command(['/usr/bin/ovs-ofctl', 'add-flow', dpname, f]) + else: + run_command(['/usr/bin/ovs-ofctl', 'add-flow', dpname, flow]) def post(self): for p in self._extra_ports: @@ -667,3 +714,17 @@ class DatapathVswitch(Datapath): netdev_down(p) datapath_modify_config(vsctl_argv) + +# +# utility methods +# + +def vswitchCfgQuery(action_args): + cmd = ['%s/usr/bin/ovs-vsctl' % root_prefix(), + '--timeout=5', '-vANY:console:emer'] + action_args + output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate() + if len(output) == 0 or output[0] == None: + output = "" + else: + output = output[0].strip() + return output