1 # Copyright (c) 2008,2009,2011 Citrix Systems, Inc.
2 # Copyright (c) 2009,2010,2011,2012 Nicira, Inc.
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU Lesser General Public License as published
6 # by the Free Software Foundation; version 2.1 only. with the special
7 # exception on linking described in file LICENSE.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Lesser General Public License for more details.
14 from InterfaceReconfigure import *
20 # Bare Network Devices -- network devices without IP configuration
23 def netdev_down(netdev):
24 """Bring down a bare network device"""
25 if not netdev_exists(netdev):
26 log("netdev: down: device %s does not exist, ignoring" % netdev)
28 run_command(["/sbin/ifconfig", netdev, 'down'])
30 def netdev_up(netdev, mtu=None):
31 """Bring up a bare network device"""
32 if not netdev_exists(netdev):
33 raise Error("netdev: up: device %s does not exist" % netdev)
40 run_command(["/sbin/ifconfig", netdev, 'up'] + mtu)
42 # This is a list of drivers that do support VLAN tx or rx acceleration, but
43 # to which the VLAN bug workaround should not be applied. This could be
44 # because these are known-good drivers (that is, they do not have any of
45 # the bugs that the workaround avoids) or because the VLAN bug workaround
46 # will not work for them and may cause other problems.
48 # This is a very short list because few drivers have been tested.
49 NO_VLAN_WORKAROUND_DRIVERS = (
52 def netdev_get_driver_name(netdev):
53 """Returns the name of the driver for network device 'netdev'"""
54 symlink = '%s/sys/class/net/%s/device/driver' % (root_prefix(), netdev)
56 target = os.readlink(symlink)
58 log("%s: could not read netdev's driver name (%s)" % (netdev, e))
61 slash = target.rfind('/')
63 log("target %s of symbolic link %s does not contain slash"
67 return target[slash + 1:]
69 def netdev_get_features(netdev):
70 """Returns the features bitmap for the driver for 'netdev'.
71 The features bitmap is a set of NETIF_F_ flags supported by its driver."""
73 features = open("%s/sys/class/net/%s/features" % (root_prefix(), netdev)).read().strip()
74 return int(features, 0)
76 return 0 # interface prolly doesn't exist
78 def netdev_has_vlan_accel(netdev):
79 """Returns True if 'netdev' supports VLAN acceleration, False otherwise."""
80 NETIF_F_HW_VLAN_TX = 128
81 NETIF_F_HW_VLAN_RX = 256
82 NETIF_F_VLAN = NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX
83 return (netdev_get_features(netdev) & NETIF_F_VLAN) != 0
89 def pif_currently_in_use(pif):
90 """Determine if a PIF is currently in use.
92 A PIF is determined to be currently in use if
93 - PIF.currently-attached is true
94 - Any bond master is currently attached
95 - Any VLAN master is currently attached
97 rec = db().get_pif_record(pif)
98 if rec['currently_attached']:
99 log("configure_datapath: %s is currently attached" % (pif_netdev_name(pif)))
101 for b in pif_get_bond_masters(pif):
102 if pif_currently_in_use(b):
103 log("configure_datapath: %s is in use by BOND master %s" % (pif_netdev_name(pif),pif_netdev_name(b)))
105 for v in pif_get_vlan_masters(pif):
106 if pif_currently_in_use(v):
107 log("configure_datapath: %s is in use by VLAN master %s" % (pif_netdev_name(pif),pif_netdev_name(v)))
112 # Datapath Configuration
115 def pif_datapath(pif):
116 """Return the datapath PIF associated with PIF.
117 A non-VLAN PIF is its own datapath PIF, except that a bridgeless PIF has
118 no datapath PIF at all.
119 A VLAN PIF's datapath PIF is its VLAN slave's datapath PIF.
122 return pif_datapath(pif_get_vlan_slave(pif))
124 pifrec = db().get_pif_record(pif)
125 nwrec = db().get_network_record(pifrec['network'])
126 if not nwrec['bridge']:
131 def datapath_get_physical_pifs(pif):
132 """Return the PIFs for the physical network device(s) associated with a datapath PIF.
133 For a bond master PIF, these are the bond slave PIFs.
134 For a non-VLAN, non-bond master PIF, the PIF is its own physical device PIF.
136 A VLAN PIF cannot be a datapath PIF.
138 if pif_is_tunnel(pif):
140 elif pif_is_vlan(pif):
141 # Seems like overkill...
142 raise Error("get-physical-pifs should not get passed a VLAN")
143 elif pif_is_bond(pif):
144 return pif_get_bond_slaves(pif)
148 def datapath_deconfigure_physical(netdev):
149 return ['--', '--with-iface', '--if-exists', 'del-port', netdev]
158 raise Error("strings may not contain null bytes")
172 return r'\x%02x' % ord(c)
173 return '"' + re.sub(r'["\\\000-\037]', escape, s) + '"'
175 def datapath_configure_tunnel(pif):
178 def datapath_configure_bond(pif,slaves):
179 bridge = pif_bridge_name(pif)
180 pifrec = db().get_pif_record(pif)
181 interface = pif_netdev_name(pif)
183 argv = ['--', '--fake-iface', 'add-bond', bridge, interface]
185 argv += [pif_netdev_name(slave)]
189 "mode": "balance-slb",
194 "hashing-algorithm": "src_mac",
196 # override defaults with values from other-config whose keys
198 oc = pifrec['other_config']
199 overrides = filter(lambda (key,val):
200 key.startswith("bond-"), oc.items())
201 overrides = map(lambda (key,val): (key[5:], val), overrides)
202 bond_options.update(overrides)
206 argv += ['--', 'set', 'Port', interface]
207 if pifrec['MAC'] != "":
208 argv += ['MAC=%s' % vsctl_escape(pifrec['MAC'])]
209 for (name,val) in bond_options.items():
210 if name in ['updelay', 'downdelay']:
211 # updelay and downdelay have dedicated schema columns.
212 # The value must be a nonnegative integer.
218 argv += ['bond_%s=%d' % (name, value)]
220 log("bridge %s has invalid %s '%s'" % (bridge, name, value))
221 elif name in ['miimon', 'use_carrier']:
227 if name == 'use_carrier':
232 argv += ["other-config:bond-detect-mode=%s" % value]
234 argv += ["other-config:bond-miimon-interval=%d" % value]
236 log("bridge %s has invalid %s '%s'" % (bridge, name, value))
239 elif name == "hashing-algorithm":
242 # Pass other bond options into other_config.
243 argv += ["other-config:%s=%s" % (vsctl_escape("bond-%s" % name),
247 argv += ['lacp=active']
249 if halgo == 'src_mac':
250 argv += ['bond_mode=balance-slb']
251 elif halgo == "tcpudp_ports":
252 argv += ['bond_mode=balance-tcp']
254 log("bridge %s has invalid bond-hashing-algorithm '%s'" % (bridge, halgo))
255 argv += ['bond_mode=balance-slb']
256 elif mode in ['balance-slb', 'active-backup']:
257 argv += ['lacp=off', 'bond_mode=%s' % mode]
259 log("bridge %s has invalid bond-mode '%s'" % (bridge, mode))
260 argv += ['lacp=off', 'bond_mode=balance-slb']
264 def datapath_deconfigure_bond(netdev):
265 return ['--', '--with-iface', '--if-exists', 'del-port', netdev]
267 def datapath_deconfigure_ipdev(interface):
268 return ['--', '--with-iface', '--if-exists', 'del-port', interface]
270 def datapath_modify_config(commands):
271 #log("modifying configuration:")
275 rc = run_command(['/usr/bin/ovs-vsctl'] + ['--timeout=20']
276 + [c for c in commands if not c.startswith('#')])
278 raise Error("Failed to modify vswitch configuration")
282 # Toplevel Datapath Configuration.
285 def configure_datapath(pif):
286 """Bring up the configuration for 'pif', which must not be a VLAN PIF, by:
287 - Tearing down other PIFs that use the same physical devices as 'pif'.
288 - Ensuring that 'pif' itself is set up.
289 - *Not* tearing down any PIFs that are stacked on top of 'pif' (i.e. VLANs
292 Returns a tuple containing
293 - A list containing the necessary vsctl command line arguments
294 - A list of additional devices which should be brought up after
295 the configuration is applied.
296 - A list containing flows to apply to the pif bridge, note that
297 port numbers may need to be substituted once ofport is known
304 assert not pif_is_vlan(pif)
305 bridge = pif_bridge_name(pif)
307 physical_devices = datapath_get_physical_pifs(pif)
309 vsctl_argv += ['## configuring datapath %s' % bridge]
311 # Determine additional devices to deconfigure.
313 # Given all physical devices which are part of this PIF we need to
315 # - any additional bond which a physical device is part of.
316 # - any additional physical devices which are part of an additional bond.
318 # Any of these which are not currently in use should be brought
319 # down and deconfigured.
320 extra_down_bonds = []
321 extra_down_ports = []
322 for p in physical_devices:
323 for bond in pif_get_bond_masters(p):
325 log("configure_datapath: leaving bond %s up" % pif_netdev_name(bond))
327 if bond in extra_down_bonds:
329 if db().get_pif_record(bond)['currently_attached']:
330 log("configure_datapath: implicitly tearing down currently-attached bond %s" % pif_netdev_name(bond))
332 extra_down_bonds += [bond]
334 for s in pif_get_bond_slaves(bond):
335 if s in physical_devices:
337 if s in extra_down_ports:
339 if pif_currently_in_use(s):
341 extra_down_ports += [s]
343 log("configure_datapath: bridge - %s" % bridge)
344 log("configure_datapath: physical - %s" % [pif_netdev_name(p) for p in physical_devices])
345 log("configure_datapath: extra ports - %s" % [pif_netdev_name(p) for p in extra_down_ports])
346 log("configure_datapath: extra bonds - %s" % [pif_netdev_name(p) for p in extra_down_bonds])
348 # Need to fully deconfigure any bridge which any of the:
353 for brpif in physical_devices + extra_down_ports + extra_down_bonds:
356 b = pif_bridge_name(brpif)
360 vsctl_argv += ['# remove bridge %s' % b]
361 vsctl_argv += ['--', '--if-exists', 'del-br', b]
363 for n in extra_down_ports:
364 dev = pif_netdev_name(n)
365 vsctl_argv += ['# deconfigure sibling physical device %s' % dev]
366 vsctl_argv += datapath_deconfigure_physical(dev)
369 for n in extra_down_bonds:
370 dev = pif_netdev_name(n)
371 vsctl_argv += ['# deconfigure bond device %s' % dev]
372 vsctl_argv += datapath_deconfigure_bond(dev)
375 for p in physical_devices:
376 dev = pif_netdev_name(p)
377 vsctl_argv += ['# deconfigure physical port %s' % dev]
378 vsctl_argv += datapath_deconfigure_physical(dev)
380 vsctl_argv += ['--', '--may-exist', 'add-br', bridge]
382 if len(physical_devices) > 1:
383 vsctl_argv += ['# deconfigure bond %s' % pif_netdev_name(pif)]
384 vsctl_argv += datapath_deconfigure_bond(pif_netdev_name(pif))
385 vsctl_argv += ['# configure bond %s' % pif_netdev_name(pif)]
386 vsctl_argv += datapath_configure_bond(pif, physical_devices)
387 extra_up_ports += [pif_netdev_name(pif)]
388 elif len(physical_devices) == 1:
389 iface = pif_netdev_name(physical_devices[0])
390 vsctl_argv += ['# add physical device %s' % iface]
391 vsctl_argv += ['--', '--may-exist', 'add-port', bridge, iface]
392 elif pif_is_tunnel(pif):
393 datapath_configure_tunnel(pif)
395 vsctl_argv += ['# configure Bridge MAC']
396 vsctl_argv += ['--', 'set', 'Bridge', bridge,
397 'other-config:hwaddr=%s' % vsctl_escape(db().get_pif_record(pif)['MAC'])]
399 pool = db().get_pool_record()
400 network = db().get_network_by_bridge(bridge)
403 valid_fail_modes = ['standalone', 'secure']
406 network_rec = db().get_network_record(network)
407 fail_mode = network_rec['other_config'].get('vswitch-controller-fail-mode')
409 if (fail_mode not in valid_fail_modes) and pool:
410 fail_mode = pool['other_config'].get('vswitch-controller-fail-mode')
411 # Add default flows to allow management traffic if fail-mode
412 # transitions to secure based on pool fail-mode setting
413 if fail_mode == 'secure' and db().get_pif_record(pif).get('management', False):
414 prev_fail_mode = vswitchCfgQuery(['get-fail-mode', bridge])
415 if prev_fail_mode != 'secure':
416 tp = 'idle_timeout=0,priority=0'
417 host_mgmt_mac = db().get_pif_record(pif)['MAC']
418 # account for bond as management interface
419 if len(physical_devices) > 1:
420 bridge_flows += ['%s,in_port=local,arp,dl_src=%s,actions=NORMAL' % (tp, host_mgmt_mac)]
421 bridge_flows += ['%s,in_port=local,dl_src=%s,actions=NORMAL' % (tp, host_mgmt_mac)]
422 # we don't know slave ofports yet, substitute later
423 bridge_flows += ['%s,in_port=%%s,arp,nw_proto=1,actions=local' % (tp)]
424 bridge_flows += ['%s,in_port=%%s,dl_dst=%s,actions=local' % (tp, host_mgmt_mac)]
426 bridge_flows += ['%s,in_port=%%s,arp,nw_proto=1,actions=local' % (tp)]
427 bridge_flows += ['%s,in_port=local,arp,dl_src=%s,actions=%%s' % (tp, host_mgmt_mac)]
428 bridge_flows += ['%s,in_port=%%s,dl_dst=%s,actions=local' % (tp, host_mgmt_mac)]
429 bridge_flows += ['%s,in_port=local,dl_src=%s,actions=%%s' % (tp, host_mgmt_mac)]
431 if fail_mode not in valid_fail_modes:
432 fail_mode = 'standalone'
434 vsctl_argv += ['--', 'set', 'Bridge', bridge, 'fail_mode=%s' % fail_mode]
437 dib = network_rec['other_config'].get('vswitch-disable-in-band')
439 vsctl_argv += ['--', 'remove', 'Bridge', bridge, 'other_config', 'disable-in-band']
440 elif dib in ['true', 'false']:
441 vsctl_argv += ['--', 'set', 'Bridge', bridge, 'other_config:disable-in-band=' + dib]
443 log('"' + dib + '"' "isn't a valid setting for other_config:disable-in-band on " + bridge)
445 vsctl_argv += set_br_external_ids(pif)
446 vsctl_argv += ['## done configuring datapath %s' % bridge]
448 return vsctl_argv,extra_up_ports,bridge_flows
450 def deconfigure_bridge(pif):
453 bridge = pif_bridge_name(pif)
455 log("deconfigure_bridge: bridge - %s" % bridge)
457 vsctl_argv += ['# deconfigure bridge %s' % bridge]
458 vsctl_argv += ['--', '--if-exists', 'del-br', bridge]
462 def set_br_external_ids(pif):
463 pifrec = db().get_pif_record(pif)
464 dp = pif_datapath(pif)
465 dprec = db().get_pif_record(dp)
467 xs_network_uuids = []
468 for nwpif in db().get_pifs_by_device(pifrec['device']):
469 rec = db().get_pif_record(nwpif)
471 # When state is read from dbcache PIF.currently_attached
472 # is always assumed to be false... Err on the side of
473 # listing even detached networks for the time being.
474 #if nwpif != pif and not rec['currently_attached']:
475 # log("Network PIF %s not currently attached (%s)" % (rec['uuid'],pifrec['uuid']))
477 nwrec = db().get_network_record(rec['network'])
480 if pif_is_vlan(nwpif):
481 xs_network_uuids.append(uuid)
483 xs_network_uuids.insert(0, uuid)
486 vsctl_argv += ['# configure xs-network-uuids']
487 vsctl_argv += ['--', 'br-set-external-id', pif_bridge_name(pif),
488 'xs-network-uuids', ';'.join(xs_network_uuids)]
496 class DatapathVswitch(Datapath):
497 def __init__(self, pif):
498 Datapath.__init__(self, pif)
499 self._dp = pif_datapath(pif)
500 self._ipdev = pif_ipdev_name(pif)
501 self._bridge_flows = []
503 if pif_is_vlan(pif) and not self._dp:
504 raise Error("Unbridged VLAN devices not implemented yet")
506 log("Configured for Vswitch datapath")
510 if not os.path.exists("/var/run/openvswitch/db.sock"):
511 # ovsdb-server is not running, so we can't update the database.
512 # Probably we are being called as part of system shutdown. Just
513 # skip the update, since the external-ids will be updated on the
518 for pif in db().get_all_pifs():
519 pifrec = db().get_pif_record(pif)
520 if not pif_is_vlan(pif) and pifrec['currently_attached']:
521 vsctl_argv += set_br_external_ids(pif)
524 datapath_modify_config(vsctl_argv)
526 def configure_ipdev(self, cfg):
527 cfg.write("TYPE=Ethernet\n")
529 def preconfigure(self, parent):
534 pifrec = db().get_pif_record(self._pif)
535 dprec = db().get_pif_record(self._dp)
538 c,e,f = configure_datapath(self._dp)
539 bridge = pif_bridge_name(self._pif)
544 dpname = pif_bridge_name(self._dp)
546 if pif_is_vlan(self._pif):
547 # In some cases XAPI may misguidedly leave an instance of
548 # 'bridge' which should be deleted.
549 vsctl_argv += ['--', '--if-exists', 'del-br', bridge]
551 # configure_datapath() set up the underlying datapath bridge.
552 # Stack a VLAN bridge on top of it.
553 vsctl_argv += ['--', '--may-exist', 'add-br',
554 bridge, dpname, pifrec['VLAN']]
556 vsctl_argv += set_br_external_ids(self._pif)
559 vsctl_argv += ["# deconfigure ipdev %s" % ipdev]
560 vsctl_argv += datapath_deconfigure_ipdev(ipdev)
561 vsctl_argv += ["# reconfigure ipdev %s" % ipdev]
562 vsctl_argv += ['--', 'add-port', bridge, ipdev]
565 vsctl_argv += ['# configure Interface MAC']
566 vsctl_argv += ['--', 'set', 'Interface', pif_ipdev_name(self._pif),
567 'MAC=%s' % vsctl_escape(dprec['MAC'])]
569 self._vsctl_argv = vsctl_argv
570 self._extra_ports = extra_ports
571 self._bridge_flows = bridge_flows
573 def bring_down_existing(self):
574 # interface-reconfigure is never explicitly called to down a
575 # bond master. However, when we are called to up a slave it
576 # is implicit that we are destroying the master. Conversely,
577 # when we are called to up a bond is is implicit that we are
578 # taking down the slaves.
580 # This is (only) important in the case where the device being
581 # implicitly taken down uses DHCP. We need to kill the
582 # dhclient process, otherwise performing the inverse operation
583 # later later will fail because ifup will refuse to start a
584 # duplicate dhclient.
585 bond_masters = pif_get_bond_masters(self._pif)
586 for master in bond_masters:
587 log("action_up: bring down bond master %s" % (pif_netdev_name(master)))
588 run_command(["/sbin/ifdown", pif_bridge_name(master)])
590 bond_slaves = pif_get_bond_slaves(self._pif)
591 for slave in bond_slaves:
592 log("action_up: bring down bond slave %s" % (pif_netdev_name(slave)))
593 run_command(["/sbin/ifdown", pif_bridge_name(slave)])
596 # Bring up physical devices. ovs-vswitchd initially enables or
597 # disables bond slaves based on whether carrier is detected
598 # when they are added, and a network device that is down
599 # always reports "no carrier".
600 physical_devices = datapath_get_physical_pifs(self._dp)
602 if pif_is_bond(self._dp):
603 brec = db().get_pif_record(self._dp)
604 bond_mtu = mtu_setting(brec['network'], "PIF", brec['other_config'])
608 for p in physical_devices:
609 prec = db().get_pif_record(p)
610 oc = prec['other_config']
612 dev = pif_netdev_name(p)
617 mtu = mtu_setting(prec['network'], "PIF", oc)
621 settings, offload = ethtool_settings(oc, PIF_OTHERCONFIG_DEFAULTS)
623 run_command(['/sbin/ethtool', '-s', dev] + settings)
625 run_command(['/sbin/ethtool', '-K', dev] + offload)
627 driver = netdev_get_driver_name(dev)
628 if 'vlan-bug-workaround' in oc:
629 vlan_bug_workaround = oc['vlan-bug-workaround'] == 'true'
630 elif driver in NO_VLAN_WORKAROUND_DRIVERS:
631 vlan_bug_workaround = False
633 vlan_bug_workaround = netdev_has_vlan_accel(dev)
635 if vlan_bug_workaround:
639 run_command(['/usr/sbin/ovs-vlan-bug-workaround', dev, setting])
641 datapath_modify_config(self._vsctl_argv)
642 if self._bridge_flows:
644 physical_devices = datapath_get_physical_pifs(self._dp)
645 if len(physical_devices) > 1:
646 for slave in physical_devices:
647 name = pif_netdev_name(slave)
648 ofport = vswitchCfgQuery(['get', 'interface', name, 'ofport'])
649 ofports.append(ofport)
651 name = pif_netdev_name(self._dp)
652 ofport = vswitchCfgQuery(['get', 'interface', name, 'ofport'])
653 ofports.append(ofport)
654 dpname = pif_bridge_name(self._dp)
655 for flow in self._bridge_flows:
656 if flow.find('in_port=%s') != -1 or flow.find('actions=%s') != -1:
659 run_command(['/usr/bin/ovs-ofctl', 'add-flow', dpname, f])
661 run_command(['/usr/bin/ovs-ofctl', 'add-flow', dpname, flow])
664 for p in self._extra_ports:
665 log("action_up: bring up %s" % p)
668 def bring_down(self):
674 bridge = pif_bridge_name(dp)
676 log("deconfigure ipdev %s on %s" % (ipdev,bridge))
677 vsctl_argv += ["# deconfigure ipdev %s" % ipdev]
678 vsctl_argv += datapath_deconfigure_ipdev(ipdev)
680 if pif_is_vlan(self._pif):
681 # Delete the VLAN bridge.
682 vsctl_argv += deconfigure_bridge(self._pif)
684 # If the VLAN's slave is attached, leave datapath setup.
685 slave = pif_get_vlan_slave(self._pif)
686 if db().get_pif_record(slave)['currently_attached']:
687 log("action_down: vlan slave is currently attached")
690 # If the VLAN's slave has other VLANs that are attached, leave datapath setup.
691 for master in pif_get_vlan_masters(slave):
692 if master != self._pif and db().get_pif_record(master)['currently_attached']:
693 log("action_down: vlan slave has other master: %s" % pif_netdev_name(master))
696 # Otherwise, take down the datapath too (fall through)
698 log("action_down: no more masters, bring down slave %s" % bridge)
700 # Stop here if this PIF has attached VLAN masters.
701 masters = [db().get_pif_record(m)['VLAN'] for m in pif_get_vlan_masters(self._pif) if db().get_pif_record(m)['currently_attached']]
703 log("Leaving datapath %s up due to currently attached VLAN masters %s" % (bridge, masters))
707 vsctl_argv += deconfigure_bridge(dp)
709 physical_devices = [pif_netdev_name(p) for p in datapath_get_physical_pifs(dp)]
711 log("action_down: bring down physical devices - %s" % physical_devices)
713 for p in physical_devices:
716 datapath_modify_config(vsctl_argv)
722 def vswitchCfgQuery(action_args):
723 cmd = ['%s/usr/bin/ovs-vsctl' % root_prefix(),
724 '--timeout=5', '-vconsole:off'] + action_args
725 output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()
726 if len(output) == 0 or output[0] == None:
729 output = output[0].strip()