1 # Copyright (c) 2008,2009,2011 Citrix Systems, Inc.
2 # Copyright (c) 2009,2010,2011 Nicira Networks.
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU Lesser General Public License as published
6 # by the Free Software Foundation; version 2.1 only. with the special
7 # exception on linking described in file LICENSE.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Lesser General Public License for more details.
14 from InterfaceReconfigure import *
19 # Bare Network Devices -- network devices without IP configuration
22 def netdev_down(netdev):
23 """Bring down a bare network device"""
24 if not netdev_exists(netdev):
25 log("netdev: down: device %s does not exist, ignoring" % netdev)
27 run_command(["/sbin/ifconfig", netdev, 'down'])
29 def netdev_up(netdev, mtu=None):
30 """Bring up a bare network device"""
31 if not netdev_exists(netdev):
32 raise Error("netdev: up: device %s does not exist" % netdev)
39 run_command(["/sbin/ifconfig", netdev, 'up'] + mtu)
41 # This is a list of drivers that do support VLAN tx or rx acceleration, but
42 # to which the VLAN bug workaround should not be applied. This could be
43 # because these are known-good drivers (that is, they do not have any of
44 # the bugs that the workaround avoids) or because the VLAN bug workaround
45 # will not work for them and may cause other problems.
47 # This is a very short list because few drivers have been tested.
48 NO_VLAN_WORKAROUND_DRIVERS = (
51 def netdev_get_driver_name(netdev):
52 """Returns the name of the driver for network device 'netdev'"""
53 symlink = '%s/sys/class/net/%s/device/driver' % (root_prefix(), netdev)
55 target = os.readlink(symlink)
57 log("%s: could not read netdev's driver name (%s)" % (netdev, e))
60 slash = target.rfind('/')
62 log("target %s of symbolic link %s does not contain slash"
66 return target[slash + 1:]
68 def netdev_get_features(netdev):
69 """Returns the features bitmap for the driver for 'netdev'.
70 The features bitmap is a set of NETIF_F_ flags supported by its driver."""
72 features = open("%s/sys/class/net/%s/features" % (root_prefix(), netdev)).read().strip()
73 return int(features, 0)
75 return 0 # interface prolly doesn't exist
77 def netdev_has_vlan_accel(netdev):
78 """Returns True if 'netdev' supports VLAN acceleration, False otherwise."""
79 NETIF_F_HW_VLAN_TX = 128
80 NETIF_F_HW_VLAN_RX = 256
81 NETIF_F_VLAN = NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX
82 return (netdev_get_features(netdev) & NETIF_F_VLAN) != 0
88 def pif_currently_in_use(pif):
89 """Determine if a PIF is currently in use.
91 A PIF is determined to be currently in use if
92 - PIF.currently-attached is true
93 - Any bond master is currently attached
94 - Any VLAN master is currently attached
96 rec = db().get_pif_record(pif)
97 if rec['currently_attached']:
98 log("configure_datapath: %s is currently attached" % (pif_netdev_name(pif)))
100 for b in pif_get_bond_masters(pif):
101 if pif_currently_in_use(b):
102 log("configure_datapath: %s is in use by BOND master %s" % (pif_netdev_name(pif),pif_netdev_name(b)))
104 for v in pif_get_vlan_masters(pif):
105 if pif_currently_in_use(v):
106 log("configure_datapath: %s is in use by VLAN master %s" % (pif_netdev_name(pif),pif_netdev_name(v)))
111 # Datapath Configuration
114 def pif_datapath(pif):
115 """Return the datapath PIF associated with PIF.
116 A non-VLAN PIF is its own datapath PIF, except that a bridgeless PIF has
117 no datapath PIF at all.
118 A VLAN PIF's datapath PIF is its VLAN slave's datapath PIF.
121 return pif_datapath(pif_get_vlan_slave(pif))
123 pifrec = db().get_pif_record(pif)
124 nwrec = db().get_network_record(pifrec['network'])
125 if not nwrec['bridge']:
130 def datapath_get_physical_pifs(pif):
131 """Return the PIFs for the physical network device(s) associated with a datapath PIF.
132 For a bond master PIF, these are the bond slave PIFs.
133 For a non-VLAN, non-bond master PIF, the PIF is its own physical device PIF.
135 A VLAN PIF cannot be a datapath PIF.
137 if pif_is_tunnel(pif):
139 elif pif_is_vlan(pif):
140 # Seems like overkill...
141 raise Error("get-physical-pifs should not get passed a VLAN")
142 elif pif_is_bond(pif):
143 return pif_get_bond_slaves(pif)
147 def datapath_deconfigure_physical(netdev):
148 return ['--', '--with-iface', '--if-exists', 'del-port', netdev]
157 raise Error("strings may not contain null bytes")
171 return r'\x%02x' % ord(c)
172 return '"' + re.sub(r'["\\\000-\037]', escape, s) + '"'
174 def datapath_configure_tunnel(pif):
177 def datapath_configure_bond(pif,slaves):
178 bridge = pif_bridge_name(pif)
179 pifrec = db().get_pif_record(pif)
180 interface = pif_netdev_name(pif)
182 argv = ['--', '--fake-iface', 'add-bond', bridge, interface]
184 argv += [pif_netdev_name(slave)]
188 "mode": "balance-slb",
193 "hashing-algorithm": "src_mac",
195 # override defaults with values from other-config whose keys
197 oc = pifrec['other_config']
198 overrides = filter(lambda (key,val):
199 key.startswith("bond-"), oc.items())
200 overrides = map(lambda (key,val): (key[5:], val), overrides)
201 bond_options.update(overrides)
205 argv += ['--', 'set', 'Port', interface]
206 if pifrec['MAC'] != "":
207 argv += ['MAC=%s' % vsctl_escape(pifrec['MAC'])]
208 for (name,val) in bond_options.items():
209 if name in ['updelay', 'downdelay']:
210 # updelay and downdelay have dedicated schema columns.
211 # The value must be a nonnegative integer.
217 argv += ['bond_%s=%d' % (name, value)]
219 log("bridge %s has invalid %s '%s'" % (bridge, name, value))
220 elif name in ['miimon', 'use_carrier']:
226 if name == 'use_carrier':
231 argv += ["other-config:bond-detect-mode=%s" % value]
233 argv += ["other-config:bond-miimon-interval=%d" % value]
235 log("bridge %s has invalid %s '%s'" % (bridge, name, value))
238 elif name == "hashing-algorithm":
241 # Pass other bond options into other_config.
242 argv += ["other-config:%s=%s" % (vsctl_escape("bond-%s" % name),
246 argv += ['lacp=active']
248 if halgo == 'src_mac':
249 argv += ['bond_mode=balance-slb']
250 elif halgo == "tcpudp_ports":
251 argv += ['bond_mode=balance-tcp']
253 log("bridge %s has invalid bond-hashing-algorithm '%s'" % (bridge, halgo))
254 argv += ['bond_mode=balance-slb']
255 elif mode in ['balance-slb', 'active-backup']:
256 argv += ['lacp=off', 'bond_mode=%s' % mode]
258 log("bridge %s has invalid bond-mode '%s'" % (bridge, mode))
259 argv += ['lacp=off', 'bond_mode=balance-slb']
263 def datapath_deconfigure_bond(netdev):
264 return ['--', '--with-iface', '--if-exists', 'del-port', netdev]
266 def datapath_deconfigure_ipdev(interface):
267 return ['--', '--with-iface', '--if-exists', 'del-port', interface]
269 def datapath_modify_config(commands):
270 #log("modifying configuration:")
274 rc = run_command(['/usr/bin/ovs-vsctl'] + ['--timeout=20']
275 + [c for c in commands if not c.startswith('#')])
277 raise Error("Failed to modify vswitch configuration")
281 # Toplevel Datapath Configuration.
284 def configure_datapath(pif):
285 """Bring up the configuration for 'pif', which must not be a VLAN PIF, by:
286 - Tearing down other PIFs that use the same physical devices as 'pif'.
287 - Ensuring that 'pif' itself is set up.
288 - *Not* tearing down any PIFs that are stacked on top of 'pif' (i.e. VLANs
291 Returns a tuple containing
292 - A list containing the necessary vsctl command line arguments
293 - A list of additional devices which should be brought up after
294 the configuration is applied.
300 assert not pif_is_vlan(pif)
301 bridge = pif_bridge_name(pif)
303 physical_devices = datapath_get_physical_pifs(pif)
305 vsctl_argv += ['## configuring datapath %s' % bridge]
307 # Determine additional devices to deconfigure.
309 # Given all physical devices which are part of this PIF we need to
311 # - any additional bond which a physical device is part of.
312 # - any additional physical devices which are part of an additional bond.
314 # Any of these which are not currently in use should be brought
315 # down and deconfigured.
316 extra_down_bonds = []
317 extra_down_ports = []
318 for p in physical_devices:
319 for bond in pif_get_bond_masters(p):
321 log("configure_datapath: leaving bond %s up" % pif_netdev_name(bond))
323 if bond in extra_down_bonds:
325 if db().get_pif_record(bond)['currently_attached']:
326 log("configure_datapath: implicitly tearing down currently-attached bond %s" % pif_netdev_name(bond))
328 extra_down_bonds += [bond]
330 for s in pif_get_bond_slaves(bond):
331 if s in physical_devices:
333 if s in extra_down_ports:
335 if pif_currently_in_use(s):
337 extra_down_ports += [s]
339 log("configure_datapath: bridge - %s" % bridge)
340 log("configure_datapath: physical - %s" % [pif_netdev_name(p) for p in physical_devices])
341 log("configure_datapath: extra ports - %s" % [pif_netdev_name(p) for p in extra_down_ports])
342 log("configure_datapath: extra bonds - %s" % [pif_netdev_name(p) for p in extra_down_bonds])
344 # Need to fully deconfigure any bridge which any of the:
349 for brpif in physical_devices + extra_down_ports + extra_down_bonds:
352 b = pif_bridge_name(brpif)
356 vsctl_argv += ['# remove bridge %s' % b]
357 vsctl_argv += ['--', '--if-exists', 'del-br', b]
359 for n in extra_down_ports:
360 dev = pif_netdev_name(n)
361 vsctl_argv += ['# deconfigure sibling physical device %s' % dev]
362 vsctl_argv += datapath_deconfigure_physical(dev)
365 for n in extra_down_bonds:
366 dev = pif_netdev_name(n)
367 vsctl_argv += ['# deconfigure bond device %s' % dev]
368 vsctl_argv += datapath_deconfigure_bond(dev)
371 for p in physical_devices:
372 dev = pif_netdev_name(p)
373 vsctl_argv += ['# deconfigure physical port %s' % dev]
374 vsctl_argv += datapath_deconfigure_physical(dev)
376 vsctl_argv += ['--', '--may-exist', 'add-br', bridge]
378 if len(physical_devices) > 1:
379 vsctl_argv += ['# deconfigure bond %s' % pif_netdev_name(pif)]
380 vsctl_argv += datapath_deconfigure_bond(pif_netdev_name(pif))
381 vsctl_argv += ['# configure bond %s' % pif_netdev_name(pif)]
382 vsctl_argv += datapath_configure_bond(pif, physical_devices)
383 extra_up_ports += [pif_netdev_name(pif)]
384 elif len(physical_devices) == 1:
385 iface = pif_netdev_name(physical_devices[0])
386 vsctl_argv += ['# add physical device %s' % iface]
387 vsctl_argv += ['--', '--may-exist', 'add-port', bridge, iface]
388 elif pif_is_tunnel(pif):
389 datapath_configure_tunnel(pif)
391 vsctl_argv += ['# configure Bridge MAC']
392 vsctl_argv += ['--', 'set', 'Bridge', bridge,
393 'other-config:hwaddr=%s' % vsctl_escape(db().get_pif_record(pif)['MAC'])]
395 pool = db().get_pool_record()
396 network = db().get_network_by_bridge(bridge)
399 valid_fail_modes = ['standalone', 'secure']
402 network_rec = db().get_network_record(network)
403 fail_mode = network_rec['other_config'].get('vswitch-controller-fail-mode')
405 if (fail_mode not in valid_fail_modes) and pool:
406 fail_mode = pool['other_config'].get('vswitch-controller-fail-mode')
408 if fail_mode not in valid_fail_modes:
409 fail_mode = 'standalone'
411 vsctl_argv += ['--', 'set', 'Bridge', bridge, 'fail_mode=%s' % fail_mode]
414 dib = network_rec['other_config'].get('vswitch-disable-in-band')
416 vsctl_argv += ['--', 'remove', 'Bridge', bridge, 'other_config', 'disable-in-band']
417 elif dib in ['true', 'false']:
418 vsctl_argv += ['--', 'set', 'Bridge', bridge, 'other_config:disable-in-band=' + dib]
420 log('"' + dib + '"' "isn't a valid setting for other_config:disable-in-band on " + bridge)
422 vsctl_argv += set_br_external_ids(pif)
423 vsctl_argv += ['## done configuring datapath %s' % bridge]
425 return vsctl_argv,extra_up_ports
427 def deconfigure_bridge(pif):
430 bridge = pif_bridge_name(pif)
432 log("deconfigure_bridge: bridge - %s" % bridge)
434 vsctl_argv += ['# deconfigure bridge %s' % bridge]
435 vsctl_argv += ['--', '--if-exists', 'del-br', bridge]
439 def set_br_external_ids(pif):
440 pifrec = db().get_pif_record(pif)
441 dp = pif_datapath(pif)
442 dprec = db().get_pif_record(dp)
444 xs_network_uuids = []
445 for nwpif in db().get_pifs_by_device(pifrec['device']):
446 rec = db().get_pif_record(nwpif)
448 # When state is read from dbcache PIF.currently_attached
449 # is always assumed to be false... Err on the side of
450 # listing even detached networks for the time being.
451 #if nwpif != pif and not rec['currently_attached']:
452 # log("Network PIF %s not currently attached (%s)" % (rec['uuid'],pifrec['uuid']))
454 nwrec = db().get_network_record(rec['network'])
457 if pif_is_vlan(nwpif):
458 xs_network_uuids.append(uuid)
460 xs_network_uuids.insert(0, uuid)
463 vsctl_argv += ['# configure xs-network-uuids']
464 vsctl_argv += ['--', 'br-set-external-id', pif_bridge_name(pif),
465 'xs-network-uuids', ';'.join(xs_network_uuids)]
473 class DatapathVswitch(Datapath):
474 def __init__(self, pif):
475 Datapath.__init__(self, pif)
476 self._dp = pif_datapath(pif)
477 self._ipdev = pif_ipdev_name(pif)
479 if pif_is_vlan(pif) and not self._dp:
480 raise Error("Unbridged VLAN devices not implemented yet")
482 log("Configured for Vswitch datapath")
486 if not os.path.exists("/var/run/openvswitch/db.sock"):
487 # ovsdb-server is not running, so we can't update the database.
488 # Probably we are being called as part of system shutdown. Just
489 # skip the update, since the external-ids will be updated on the
494 for pif in db().get_all_pifs():
495 pifrec = db().get_pif_record(pif)
496 if not pif_is_vlan(pif) and pifrec['currently_attached']:
497 vsctl_argv += set_br_external_ids(pif)
500 datapath_modify_config(vsctl_argv)
502 def configure_ipdev(self, cfg):
503 cfg.write("TYPE=Ethernet\n")
505 def preconfigure(self, parent):
509 pifrec = db().get_pif_record(self._pif)
510 dprec = db().get_pif_record(self._dp)
513 c,e = configure_datapath(self._dp)
514 bridge = pif_bridge_name(self._pif)
518 dpname = pif_bridge_name(self._dp)
520 if pif_is_vlan(self._pif):
521 # In some cases XAPI may misguidedly leave an instance of
522 # 'bridge' which should be deleted.
523 vsctl_argv += ['--', '--if-exists', 'del-br', bridge]
525 # configure_datapath() set up the underlying datapath bridge.
526 # Stack a VLAN bridge on top of it.
527 vsctl_argv += ['--', '--may-exist', 'add-br',
528 bridge, dpname, pifrec['VLAN']]
530 vsctl_argv += set_br_external_ids(self._pif)
533 vsctl_argv += ["# deconfigure ipdev %s" % ipdev]
534 vsctl_argv += datapath_deconfigure_ipdev(ipdev)
535 vsctl_argv += ["# reconfigure ipdev %s" % ipdev]
536 vsctl_argv += ['--', 'add-port', bridge, ipdev]
539 vsctl_argv += ['# configure Interface MAC']
540 vsctl_argv += ['--', 'set', 'Interface', pif_ipdev_name(self._pif),
541 'MAC=%s' % vsctl_escape(dprec['MAC'])]
543 self._vsctl_argv = vsctl_argv
544 self._extra_ports = extra_ports
546 def bring_down_existing(self):
547 # interface-reconfigure is never explicitly called to down a
548 # bond master. However, when we are called to up a slave it
549 # is implicit that we are destroying the master. Conversely,
550 # when we are called to up a bond is is implicit that we are
551 # taking down the slaves.
553 # This is (only) important in the case where the device being
554 # implicitly taken down uses DHCP. We need to kill the
555 # dhclient process, otherwise performing the inverse operation
556 # later later will fail because ifup will refuse to start a
557 # duplicate dhclient.
558 bond_masters = pif_get_bond_masters(self._pif)
559 for master in bond_masters:
560 log("action_up: bring down bond master %s" % (pif_netdev_name(master)))
561 run_command(["/sbin/ifdown", pif_bridge_name(master)])
563 bond_slaves = pif_get_bond_slaves(self._pif)
564 for slave in bond_slaves:
565 log("action_up: bring down bond slave %s" % (pif_netdev_name(slave)))
566 run_command(["/sbin/ifdown", pif_bridge_name(slave)])
569 # Bring up physical devices. ovs-vswitchd initially enables or
570 # disables bond slaves based on whether carrier is detected
571 # when they are added, and a network device that is down
572 # always reports "no carrier".
573 physical_devices = datapath_get_physical_pifs(self._dp)
575 if pif_is_bond(self._dp):
576 brec = db().get_pif_record(self._dp)
577 bond_mtu = mtu_setting(brec['network'], "PIF", brec['other_config'])
581 for p in physical_devices:
582 prec = db().get_pif_record(p)
583 oc = prec['other_config']
585 dev = pif_netdev_name(p)
590 mtu = mtu_setting(prec['network'], "PIF", oc)
594 settings, offload = ethtool_settings(oc, PIF_OTHERCONFIG_DEFAULTS)
596 run_command(['/sbin/ethtool', '-s', dev] + settings)
598 run_command(['/sbin/ethtool', '-K', dev] + offload)
600 driver = netdev_get_driver_name(dev)
601 if 'vlan-bug-workaround' in oc:
602 vlan_bug_workaround = oc['vlan-bug-workaround'] == 'true'
603 elif driver in NO_VLAN_WORKAROUND_DRIVERS:
604 vlan_bug_workaround = False
606 vlan_bug_workaround = netdev_has_vlan_accel(dev)
608 if vlan_bug_workaround:
612 run_command(['/usr/sbin/ovs-vlan-bug-workaround', dev, setting])
614 datapath_modify_config(self._vsctl_argv)
617 for p in self._extra_ports:
618 log("action_up: bring up %s" % p)
621 def bring_down(self):
627 bridge = pif_bridge_name(dp)
629 log("deconfigure ipdev %s on %s" % (ipdev,bridge))
630 vsctl_argv += ["# deconfigure ipdev %s" % ipdev]
631 vsctl_argv += datapath_deconfigure_ipdev(ipdev)
633 if pif_is_vlan(self._pif):
634 # Delete the VLAN bridge.
635 vsctl_argv += deconfigure_bridge(self._pif)
637 # If the VLAN's slave is attached, leave datapath setup.
638 slave = pif_get_vlan_slave(self._pif)
639 if db().get_pif_record(slave)['currently_attached']:
640 log("action_down: vlan slave is currently attached")
643 # If the VLAN's slave has other VLANs that are attached, leave datapath setup.
644 for master in pif_get_vlan_masters(slave):
645 if master != self._pif and db().get_pif_record(master)['currently_attached']:
646 log("action_down: vlan slave has other master: %s" % pif_netdev_name(master))
649 # Otherwise, take down the datapath too (fall through)
651 log("action_down: no more masters, bring down slave %s" % bridge)
653 # Stop here if this PIF has attached VLAN masters.
654 masters = [db().get_pif_record(m)['VLAN'] for m in pif_get_vlan_masters(self._pif) if db().get_pif_record(m)['currently_attached']]
656 log("Leaving datapath %s up due to currently attached VLAN masters %s" % (bridge, masters))
660 vsctl_argv += deconfigure_bridge(dp)
662 physical_devices = [pif_netdev_name(p) for p in datapath_get_physical_pifs(dp)]
664 log("action_down: bring down physical devices - %s" % physical_devices)
666 for p in physical_devices:
669 datapath_modify_config(vsctl_argv)