From bc17490dfccee9734a7a8f7b61bafec3a7664b8a Mon Sep 17 00:00:00 2001 From: Ben Pfaff Date: Wed, 7 Oct 2009 12:07:27 -0700 Subject: [PATCH] xenserver: Crossport "master" interface-reconfigure to "citrix". This commit copies the interface-reconfigure script from "master" into "citrix" and fixes up a few incompatibilities: the location of ovs-cfg-mod, which in master is in /usr/bin and in citrix is in /root/vswitch/bin, and the RPM spec file fragments needed to initialize the database cache. The purpose of this commit is to obtain the bug fixes that have been applied (mainly by Ian Campbell) to "master" but which are not in "citrix". It's difficult to understand the changes from this commit alone. It is more meaningful to compare the resulting files against those currently on the master branch. --- ...pt_xensource_libexec_interface-reconfigure | 2357 +++++++++-------- xenserver/vswitch-xen.spec | 37 +- 2 files changed, 1197 insertions(+), 1197 deletions(-) diff --git a/xenserver/opt_xensource_libexec_interface-reconfigure b/xenserver/opt_xensource_libexec_interface-reconfigure index 301251f7..2b12218d 100755 --- a/xenserver/opt_xensource_libexec_interface-reconfigure +++ b/xenserver/opt_xensource_libexec_interface-reconfigure @@ -5,22 +5,26 @@ # """Usage: - %(command-name)s --session --pif [up|down|rewrite] - %(command-name)s --force [up|down|rewrite ] + %(command-name)s up + %(command-name)s down + %(command-name)s [] rewrite + %(command-name)s --force up + %(command-name)s --force down + %(command-name)s --force rewrite --device= %(command-name)s --force all down - where, - = --device= --mode=dhcp - = --device= --mode=static --ip= --netmask= [--gateway=] + where is one of: + --session --pif + --pif-uuid + and is one of: + --mode=dhcp + --mode=static --ip= --netmask= [--gateway=] Options: --session A session reference to use to access the xapi DB - --pif A PIF reference. - --force-interface An interface name. Mutually exclusive with --session/--pif. - - Either both --session and --pif or just --pif-uuid. - - is either "up" or "down" or "rewrite" + --pif A PIF reference within the session. + --pif-uuid The UUID of a PIF. + --force An interface name. """ # @@ -28,7 +32,6 @@ # # --output-directory= Write configuration to . Also disables actually # raising/lowering the interfaces -# --pif-uuid A PIF UUID, use instead of --session/--pif. # # # @@ -38,40 +41,58 @@ # 3. A network may have an associated bridge, allowing vifs to be attached # 4. A network may be bridgeless (there's no point having a bridge over a storage pif) -# XXX: --force-interface=all down - -# XXX: --force-interface rewrite - -# XXX: Sometimes this leaves "orphaned" datapaths, e.g. a datapath whose -# only port is the local port. Should delete those. - -# XXX: This can leave crud in ovs-vswitchd.conf in this scenario: -# - Create bond in XenCenter. -# - Create VLAN on bond in XenCenter. -# - Attempt to delete bond in XenCenter (this will fail because there -# is a VLAN on the bond, although the error may not be reported -# until the next step) -# - Delete VLAN in XenCenter. -# - Delete bond in XenCenter. -# At this point there will still be some configuration data for the bond -# or the VLAN in ovs-vswitchd.conf. - import XenAPI import os, sys, getopt, time, signal import syslog import traceback -import time import re -import pickle import random +from xml.dom.minidom import getDOMImplementation +from xml.dom.minidom import parse as parseXML output_directory = None db = None management_pif = None -dbcache_file = "/etc/ovs-vswitch.dbcache" -vswitch_config_dir = "/etc/openvswitch" +vswitch_state_dir = "/var/lib/openvswitch/" +dbcache_file = vswitch_state_dir + "dbcache" + +# +# Debugging and Logging. +# + +def debug_mode(): + return output_directory is not None + +def log(s): + if debug_mode(): + print >>sys.stderr, s + else: + syslog.syslog(s) + +def log_pif_action(action, pif): + pifrec = db.get_pif_record(pif) + rec = {} + rec['uuid'] = pifrec['uuid'] + rec['ip_configuration_mode'] = pifrec['ip_configuration_mode'] + rec['action'] = action + rec['pif_netdev_name'] = pif_netdev_name(pif) + rec['message'] = "Bring %(action)s PIF %(uuid)s" % rec + log("%(message)s: %(pif_netdev_name)s configured as %(ip_configuration_mode)s" % rec) + + +def run_command(command): + log("Running command: " + ' '.join(command)) + rc = os.spawnl(os.P_WAIT, command[0], *command) + if rc != 0: + log("Command failed %d: " % rc + ' '.join(command)) + return False + return True + +# +# Exceptions. +# class Usage(Exception): def __init__(self, msg): @@ -83,6 +104,10 @@ class Error(Exception): Exception.__init__(self) self.msg = msg +# +# Configuration File Handling. +# + class ConfigurationFile(object): """Write a file, tracking old and new versions. @@ -93,23 +118,23 @@ class ConfigurationFile(object): __STATE = {"OPEN":"OPEN", "NOT-APPLIED":"NOT-APPLIED", "APPLIED":"APPLIED", "REVERTED":"REVERTED", "COMMITTED": "COMMITTED"} - + def __init__(self, fname, path="/etc/sysconfig/network-scripts"): self.__state = self.__STATE['OPEN'] self.__fname = fname self.__children = [] - + if debug_mode(): dirname = output_directory else: dirname = path - + self.__path = os.path.join(dirname, fname) self.__oldpath = os.path.join(dirname, "." + fname + ".xapi-old") self.__newpath = os.path.join(dirname, "." + fname + ".xapi-new") self.__unlink = False - + self.__f = open(self.__newpath, "w") def attach_child(self, child): @@ -123,7 +148,7 @@ class ConfigurationFile(object): return open(self.path()).readlines() except: return "" - + def write(self, args): if self.__state != self.__STATE['OPEN']: raise Error("Attempt to write to file in state %s" % self.__state) @@ -135,11 +160,11 @@ class ConfigurationFile(object): self.__unlink = True self.__f.close() self.__state = self.__STATE['NOT-APPLIED'] - + def close(self): if self.__state != self.__STATE['OPEN']: raise Error("Attempt to close file in state %s" % self.__state) - + self.__f.close() self.__state = self.__STATE['NOT-APPLIED'] @@ -172,7 +197,7 @@ class ConfigurationFile(object): if not self.__unlink: os.link(self.__newpath, self.__path) else: - pass # implicit unlink of original file + pass # implicit unlink of original file # Remove temporary file. os.unlink(self.__newpath) @@ -203,9 +228,9 @@ class ConfigurationFile(object): os.unlink(self.__oldpath) # Leave .*.xapi-new as an aid to debugging. - + self.__state = self.__STATE['REVERTED'] - + def commit(self): if self.__state != self.__STATE['APPLIED']: raise Error("Attempt to commit configuration from state %s" % self.__state) @@ -214,7 +239,7 @@ class ConfigurationFile(object): child.commit() log("Committing changes to %s configuration" % self.__fname) - + if os.access(self.__oldpath, os.F_OK): os.unlink(self.__oldpath) if os.access(self.__newpath, os.F_OK): @@ -222,61 +247,225 @@ class ConfigurationFile(object): self.__state = self.__STATE['COMMITTED'] -def debug_mode(): - return output_directory is not None +# +# Helper functions for encoding/decoding database attributes to/from XML. +# -def log(s): - if debug_mode(): - print >>sys.stderr, s +def str_to_xml(xml, parent, tag, val): + e = xml.createElement(tag) + parent.appendChild(e) + v = xml.createTextNode(val) + e.appendChild(v) +def str_from_xml(n): + def getText(nodelist): + rc = "" + for node in nodelist: + if node.nodeType == node.TEXT_NODE: + rc = rc + node.data + return rc + return getText(n.childNodes).strip() + +def bool_to_xml(xml, parent, tag, val): + if val: + str_to_xml(xml, parent, tag, "True") else: - syslog.syslog(s) + str_to_xml(xml, parent, tag, "False") +def bool_from_xml(n): + s = str_from_xml(n) + if s == "True": + return True + elif s == "False": + return False + else: + raise Error("Unknown boolean value %s" % s) + +def strlist_to_xml(xml, parent, ltag, itag, val): + e = xml.createElement(ltag) + parent.appendChild(e) + for v in val: + c = xml.createElement(itag) + e.appendChild(c) + cv = xml.createTextNode(v) + c.appendChild(cv) +def strlist_from_xml(n, ltag, itag): + ret = [] + for n in n.childNodes: + if n.nodeName == itag: + ret.append(str_from_xml(n)) + return ret + +def otherconfig_to_xml(xml, parent, val, attrs): + otherconfig = xml.createElement("other_config") + parent.appendChild(otherconfig) + for n,v in val.items(): + if not n in attrs: + raise Error("Unknown other-config attribute: %s" % n) + str_to_xml(xml, otherconfig, n, v) +def otherconfig_from_xml(n, attrs): + ret = {} + for n in n.childNodes: + if n.nodeName in attrs: + ret[n.nodeName] = str_from_xml(n) + return ret -def check_allowed(pif): - pifrec = db.get_pif_record(pif) - try: - f = open("/proc/ardence") - macline = filter(lambda x: x.startswith("HWaddr:"), f.readlines()) - f.close() - if len(macline) == 1: - p = re.compile(".*\s%(MAC)s\s.*" % pifrec, re.IGNORECASE) - if p.match(macline[0]): - log("Skipping PVS device %(device)s (%(MAC)s)" % pifrec) - return False - except IOError: - pass - return True +# +# Definitions of the database objects (and their attributes) used by interface-reconfigure. +# +# Each object is defined by a dictionary mapping an attribute name in +# the xapi database to a tuple containing two items: +# - a function which takes this attribute and encodes it as XML. +# - a function which takes XML and decocdes it into a value. +# +# other-config attributes are specified as a simple array of strings + +PIF_XML_TAG = "pif" +VLAN_XML_TAG = "vlan" +BOND_XML_TAG = "bond" +NETWORK_XML_TAG = "network" + +ETHTOOL_OTHERCONFIG_ATTRS = ['ethtool-%s' % x for x in 'autoneg', 'speed', 'duplex', 'rx', 'tx', 'sg', 'tso', 'ufo', 'gso' ] + +PIF_OTHERCONFIG_ATTRS = [ 'domain', 'peerdns', 'defaultroute', 'mtu', 'static-routes' ] + \ + [ 'bond-%s' % x for x in 'mode', 'miimon', 'downdelay', 'updelay', 'use_carrier' ] + \ + ETHTOOL_OTHERCONFIG_ATTRS + +PIF_ATTRS = { 'uuid': (str_to_xml,str_from_xml), + 'management': (bool_to_xml,bool_from_xml), + 'network': (str_to_xml,str_from_xml), + 'device': (str_to_xml,str_from_xml), + 'bond_master_of': (lambda x, p, t, v: strlist_to_xml(x, p, 'bond_master_of', 'slave', v), + lambda n: strlist_from_xml(n, 'bond_master_of', 'slave')), + 'bond_slave_of': (str_to_xml,str_from_xml), + 'VLAN': (str_to_xml,str_from_xml), + 'VLAN_master_of': (str_to_xml,str_from_xml), + 'VLAN_slave_of': (lambda x, p, t, v: strlist_to_xml(x, p, 'VLAN_slave_of', 'master', v), + lambda n: strlist_from_xml(n, 'VLAN_slave_Of', 'master')), + 'ip_configuration_mode': (str_to_xml,str_from_xml), + 'IP': (str_to_xml,str_from_xml), + 'netmask': (str_to_xml,str_from_xml), + 'gateway': (str_to_xml,str_from_xml), + 'DNS': (str_to_xml,str_from_xml), + 'MAC': (str_to_xml,str_from_xml), + 'other_config': (lambda x, p, t, v: otherconfig_to_xml(x, p, v, PIF_OTHERCONFIG_ATTRS), + lambda n: otherconfig_from_xml(n, PIF_OTHERCONFIG_ATTRS)), + + # Special case: We write the current value + # PIF.currently-attached to the cache but since it will + # not be valid when we come to use the cache later + # (i.e. after a reboot) we always read it as False. + 'currently_attached': (bool_to_xml, lambda n: False), + } + +VLAN_ATTRS = { 'uuid': (str_to_xml,str_from_xml), + 'tagged_PIF': (str_to_xml,str_from_xml), + 'untagged_PIF': (str_to_xml,str_from_xml), + } + +BOND_ATTRS = { 'uuid': (str_to_xml,str_from_xml), + 'master': (str_to_xml,str_from_xml), + 'slaves': (lambda x, p, t, v: strlist_to_xml(x, p, 'slaves', 'slave', v), + lambda n: strlist_from_xml(n, 'slaves', 'slave')), + } + +NETWORK_OTHERCONFIG_ATTRS = [ 'mtu', 'static-routes' ] + ETHTOOL_OTHERCONFIG_ATTRS + +NETWORK_ATTRS = { 'uuid': (str_to_xml,str_from_xml), + 'bridge': (str_to_xml,str_from_xml), + 'PIFs': (lambda x, p, t, v: strlist_to_xml(x, p, 'PIFs', 'PIF', v), + lambda n: strlist_from_xml(n, 'PIFs', 'PIF')), + 'other_config': (lambda x, p, t, v: otherconfig_to_xml(x, p, v, NETWORK_OTHERCONFIG_ATTRS), + lambda n: otherconfig_from_xml(n, NETWORK_OTHERCONFIG_ATTRS)), + } -def interface_exists(i): - return os.path.exists("/sys/class/net/" + i) +class DatabaseCache(object): + def __read_xensource_inventory(self): + filename = "/etc/xensource-inventory" + f = open(filename, "r") + lines = [x.strip("\n") for x in f.readlines()] + f.close() -def get_netdev_mac(device): - try: - return read_first_line_of_file("/sys/class/net/%s/address" % device) - except: - # Probably no such device. - return None + defs = [ (l[:l.find("=")], l[(l.find("=") + 1):]) for l in lines ] + defs = [ (a, b.strip("'")) for (a,b) in defs ] -def get_netdev_tx_queue_len(device): - try: - return int(read_first_line_of_file("/sys/class/net/%s/tx_queue_len" - % device)) - except: - # Probably no such device. - return None + return dict(defs) + def __pif_on_host(self,pif): + return self.__pifs.has_key(pif) -def get_netdev_by_mac(mac): - for device in os.listdir("/sys/class/net"): - dev_mac = get_netdev_mac(device) - if (dev_mac and mac.lower() == dev_mac.lower() and - get_netdev_tx_queue_len(device)): - return device - return None + def __get_pif_records_from_xapi(self, session, host): + self.__pifs = {} + for (p,rec) in session.xenapi.PIF.get_all_records().items(): + if rec['host'] != host: + continue + self.__pifs[p] = {} + for f in PIF_ATTRS: + self.__pifs[p][f] = rec[f] + self.__pifs[p]['other_config'] = {} + for f in PIF_OTHERCONFIG_ATTRS: + if not rec['other_config'].has_key(f): continue + self.__pifs[p]['other_config'][f] = rec['other_config'][f] + + def __get_vlan_records_from_xapi(self, session): + self.__vlans = {} + for v in session.xenapi.VLAN.get_all(): + rec = session.xenapi.VLAN.get_record(v) + if not self.__pif_on_host(rec['untagged_PIF']): + continue + self.__vlans[v] = {} + for f in VLAN_ATTRS: + self.__vlans[v][f] = rec[f] + + def __get_bond_records_from_xapi(self, session): + self.__bonds = {} + for b in session.xenapi.Bond.get_all(): + rec = session.xenapi.Bond.get_record(b) + if not self.__pif_on_host(rec['master']): + continue + self.__bonds[b] = {} + for f in BOND_ATTRS: + self.__bonds[b][f] = rec[f] + + def __get_network_records_from_xapi(self, session): + self.__networks = {} + for n in session.xenapi.network.get_all(): + rec = session.xenapi.network.get_record(n) + self.__networks[n] = {} + for f in NETWORK_ATTRS: + if f == "PIFs": + # drop PIFs on other hosts + self.__networks[n][f] = [p for p in rec[f] if self.__pif_on_host(p)] + else: + self.__networks[n][f] = rec[f] + self.__networks[n]['other_config'] = {} + for f in NETWORK_OTHERCONFIG_ATTRS: + if not rec['other_config'].has_key(f): continue + self.__networks[n]['other_config'][f] = rec['other_config'][f] + + def __to_xml(self, xml, parent, key, ref, rec, attrs): + """Encode a database object as XML""" + e = xml.createElement(key) + parent.appendChild(e) + if ref: + e.setAttribute('ref', ref) + + for n,v in rec.items(): + if attrs.has_key(n): + h,_ = attrs[n] + h(xml, e, n, v) + else: + raise Error("Unknown attribute %s" % n) + def __from_xml(self, e, attrs): + """Decode a database object from XML""" + ref = e.attributes['ref'].value + rec = {} + for n in e.childNodes: + if n.nodeName in attrs: + _,h = attrs[n.nodeName] + rec[n.nodeName] = h(n) + return (ref,rec) -class DatabaseCache(object): def __init__(self, session_ref=None, cache_file=None): if session_ref and cache_file: raise Error("can't specify session reference and cache file") - if cache_file == None: session = XenAPI.xapi_local() @@ -287,32 +476,70 @@ class DatabaseCache(object): session._session = session_ref try: - self.__vlans = session.xenapi.VLAN.get_all_records() - self.__bonds = session.xenapi.Bond.get_all_records() - self.__pifs = session.xenapi.PIF.get_all_records() - self.__networks = session.xenapi.network.get_all_records() + + inventory = self.__read_xensource_inventory() + assert(inventory.has_key('INSTALLATION_UUID')) + log("host uuid is %s" % inventory['INSTALLATION_UUID']) + + host = session.xenapi.host.get_by_uuid(inventory['INSTALLATION_UUID']) + + self.__get_pif_records_from_xapi(session, host) + + self.__get_vlan_records_from_xapi(session) + self.__get_bond_records_from_xapi(session) + self.__get_network_records_from_xapi(session) finally: if not session_ref: session.xenapi.session.logout() else: log("Loading xapi database cache from %s" % cache_file) - f = open(cache_file, 'r') - members = pickle.load(f) - self.extras = pickle.load(f) - f.close() - self.__vlans = members['vlans'] - self.__bonds = members['bonds'] - self.__pifs = members['pifs'] - self.__networks = members['networks'] + xml = parseXML(cache_file) + + self.__pifs = {} + self.__bonds = {} + self.__vlans = {} + self.__networks = {} + + assert(len(xml.childNodes) == 1) + toplevel = xml.childNodes[0] + + assert(toplevel.nodeName == "xenserver-network-configuration") + + for n in toplevel.childNodes: + if n.nodeName == "#text": + pass + elif n.nodeName == PIF_XML_TAG: + (ref,rec) = self.__from_xml(n, PIF_ATTRS) + self.__pifs[ref] = rec + elif n.nodeName == BOND_XML_TAG: + (ref,rec) = self.__from_xml(n, BOND_ATTRS) + self.__bonds[ref] = rec + elif n.nodeName == VLAN_XML_TAG: + (ref,rec) = self.__from_xml(n, VLAN_ATTRS) + self.__vlans[ref] = rec + elif n.nodeName == NETWORK_XML_TAG: + (ref,rec) = self.__from_xml(n, NETWORK_ATTRS) + self.__networks[ref] = rec + else: + raise Error("Unknown XML element %s" % n.nodeName) + + def save(self, cache_file): + + xml = getDOMImplementation().createDocument( + None, "xenserver-network-configuration", None) + for (ref,rec) in self.__pifs.items(): + self.__to_xml(xml, xml.documentElement, PIF_XML_TAG, ref, rec, PIF_ATTRS) + for (ref,rec) in self.__bonds.items(): + self.__to_xml(xml, xml.documentElement, BOND_XML_TAG, ref, rec, BOND_ATTRS) + for (ref,rec) in self.__vlans.items(): + self.__to_xml(xml, xml.documentElement, VLAN_XML_TAG, ref, rec, VLAN_ATTRS) + for (ref,rec) in self.__networks.items(): + self.__to_xml(xml, xml.documentElement, NETWORK_XML_TAG, ref, rec, + NETWORK_ATTRS) - def save(self, cache_file, extras): f = open(cache_file, 'w') - pickle.dump({'vlans': self.__vlans, - 'bonds': self.__bonds, - 'pifs': self.__pifs, - 'networks': self.__networks}, f) - pickle.dump(extras, f) + f.write(xml.toprettyxml()) f.close() def get_pif_by_uuid(self, uuid): @@ -326,70 +553,46 @@ class DatabaseCache(object): return pifs[0] - def get_pifs_by_record(self, record): - """record is partial pif record. - Get the pif(s) whose record matches. - """ - def match(pifrec): - for key in record: - if record[key] != pifrec[key]: - return False - return True - + def get_pifs_by_device(self, device): return map(lambda (ref,rec): ref, - filter(lambda (ref,rec): match(rec), + filter(lambda (ref,rec): rec['device'] == device, self.__pifs.items())) - def get_pif_by_record(self, record): - """record is partial pif record. - Get the pif whose record matches. - """ - pifs = self.get_pifs_by_record(record) - if len(pifs) == 0: - raise Error("No matching PIF \"%s\"" % str(record)) - elif len(pifs) > 1: - raise Error("Multiple matching PIFs \"%s\"" % str(record)) - - return pifs[0] - - def get_pif_by_bridge(self, host, bridge): + def get_pif_by_bridge(self, bridge): networks = map(lambda (ref,rec): ref, filter(lambda (ref,rec): rec['bridge'] == bridge, self.__networks.items())) if len(networks) == 0: - raise Error("No matching network \"%s\"") + raise Error("No matching network \"%s\"" % bridge) answer = None for network in networks: nwrec = self.get_network_record(network) for pif in nwrec['PIFs']: pifrec = self.get_pif_record(pif) - if pifrec['host'] != host: - continue if answer: - raise Error("Multiple PIFs on %s for network %s" % (host, bridge)) + raise Error("Multiple PIFs on host for network %s" % (bridge)) answer = pif if not answer: - raise Error("No PIF on %s for network %s" % (host, bridge)) + raise Error("No PIF on host for network %s" % (bridge)) return answer def get_pif_record(self, pif): if self.__pifs.has_key(pif): return self.__pifs[pif] - raise Error("Unknown PIF \"%s\"" % pif) + raise Error("Unknown PIF \"%s\" (get_pif_record)" % pif) def get_all_pifs(self): return self.__pifs def pif_exists(self, pif): return self.__pifs.has_key(pif) - - def get_management_pif(self, host): + + def get_management_pif(self): """ Returns the management pif on host """ all = self.get_all_pifs() - for pif in all: + for pif in all: pifrec = self.get_pif_record(pif) - if pifrec['management'] and pifrec['host'] == host : - return pif + if pifrec['management']: return pif return None def get_network_record(self, network): @@ -404,192 +607,126 @@ class DatabaseCache(object): return self.__bonds[bond] else: return None - + def get_vlan_record(self, vlan): if self.__vlans.has_key(vlan): return self.__vlans[vlan] else: return None - -def bridge_name(pif): - """Return the bridge name associated with pif, or None if network is bridgeless""" - pifrec = db.get_pif_record(pif) - nwrec = db.get_network_record(pifrec['network']) - - if nwrec['bridge']: - # TODO: sanity check that nwrec['bridgeless'] != 'true' - return nwrec['bridge'] - else: - # TODO: sanity check that nwrec['bridgeless'] == 'true' - return None -def interface_name(pif): - """Construct an interface name from the given PIF record.""" - - pifrec = db.get_pif_record(pif) +# +# Boot from Network filesystem or device. +# - if pifrec['VLAN'] == '-1': - return pifrec['device'] - else: - return "%(device)s.%(VLAN)s" % pifrec +def check_allowed(pif): + """Determine whether interface-reconfigure should be manipulating this PIF. -def datapath_name(pif): - """Return the OpenFlow datapath name associated with pif. -For a non-VLAN PIF, the datapath name is the bridge name. -For a VLAN PIF, the datapath name is the bridge name for the PIF's VLAN slave. -(xapi will create a datapath named with the bridge name even though we won't -use it.) -""" + Used to prevent system PIFs (such as network root disk) from being interfered with. + """ pifrec = db.get_pif_record(pif) + try: + f = open("/proc/ardence") + macline = filter(lambda x: x.startswith("HWaddr:"), f.readlines()) + f.close() + if len(macline) == 1: + p = re.compile(".*\s%(MAC)s\s.*" % pifrec, re.IGNORECASE) + if p.match(macline[0]): + log("Skipping PVS device %(device)s (%(MAC)s)" % pifrec) + return False + except IOError: + pass + return True - if pifrec['VLAN'] == '-1': - return bridge_name(pif) - else: - return bridge_name(get_vlan_slave_of_pif(pif)) - -def ipdev_name(pif): - """Return the the name of the network device that carries the -IP configuration (if any) associated with pif. -The ipdev name is the same as the bridge name. -""" +# +# Bare Network Devices -- network devices without IP configuration +# - pifrec = db.get_pif_record(pif) - return bridge_name(pif) +def netdev_exists(netdev): + return os.path.exists("/sys/class/net/" + netdev) -def get_physdev_pifs(pif): - """Return the PIFs for the physical network device(s) associated with pif. -For a VLAN PIF, this is the VLAN slave's physical device PIF. -For a bond master PIF, these are the bond slave PIFs. -For a non-VLAN, non-bond master PIF, the PIF is its own physical device PIF. -""" +def pif_netdev_name(pif): + """Get the netdev name for a PIF.""" pifrec = db.get_pif_record(pif) - if pifrec['VLAN'] != '-1': - return get_physdev_pifs(get_vlan_slave_of_pif(pif)) - elif len(pifrec['bond_master_of']) != 0: - return get_bond_slaves_of_pif(pif) + if pif_is_vlan(pif): + return "%(device)s.%(VLAN)s" % pifrec else: - return [pif] + return pifrec['device'] -def get_physdev_names(pif): - """Return the name(s) of the physical network device(s) associated with pif. -For a VLAN PIF, the physical devices are the VLAN slave's physical devices. -For a bond master PIF, the physical devices are the bond slaves. -For a non-VLAN, non-bond master PIF, the physical device is the PIF itself. -""" +def netdev_down(netdev): + """Bring down a bare network device""" + if debug_mode(): + return + if not netdev_exists(netdev): + log("netdev: down: device %s does not exist, ignoring" % netdev) + return + run_command(["/sbin/ifconfig", netdev, 'down']) - return [db.get_pif_record(phys)['device'] for phys in get_physdev_pifs(pif)] +def netdev_up(netdev, mtu=None): + """Bring up a bare network device""" + if debug_mode(): + return + if not netdev_exists(netdev): + raise Error("netdev: up: device %s does not exist" % netdev) -def log_pif_action(action, pif): - pifrec = db.get_pif_record(pif) - pifrec['action'] = action - pifrec['interface-name'] = interface_name(pif) - if action == "rewrite": - pifrec['message'] = "Rewrite PIF %(uuid)s configuration" % pifrec + if mtu: + mtu = ["mtu", mtu] else: - pifrec['message'] = "Bring %(action)s PIF %(uuid)s" % pifrec - log("%(message)s: %(interface-name)s configured as %(ip_configuration_mode)s" % pifrec) - -def get_bond_masters_of_pif(pif): - """Returns a list of PIFs which are bond masters of this PIF""" - - pifrec = db.get_pif_record(pif) - - bso = pifrec['bond_slave_of'] - - # bond-slave-of is currently a single reference but in principle a - # PIF could be a member of several bonds which are not - # concurrently attached. Be robust to this possibility. - if not bso or bso == "OpaqueRef:NULL": - bso = [] - elif not type(bso) == list: - bso = [bso] - - bondrecs = [db.get_bond_record(bond) for bond in bso] - bondrecs = [rec for rec in bondrecs if rec] - - return [bond['master'] for bond in bondrecs] - -def get_bond_slaves_of_pif(pif): - """Returns a list of PIFs which make up the given bonded pif.""" - - pifrec = db.get_pif_record(pif) - host = pifrec['host'] - - bmo = pifrec['bond_master_of'] - if len(bmo) > 1: - raise Error("Bond-master-of contains too many elements") - - if len(bmo) == 0: - return [] - - bondrec = db.get_bond_record(bmo[0]) - if not bondrec: - raise Error("No bond record for bond master PIF") - - return bondrec['slaves'] - -def get_vlan_slave_of_pif(pif): - """Find the PIF which is the VLAN slave of pif. + mtu = [] + + run_command(["/sbin/ifconfig", netdev, 'up'] + mtu) -Returns the 'physical' PIF underneath the a VLAN PIF @pif.""" +def netdev_remap_name(pif, already_renamed=[]): + """Check whether 'pif' exists and has the correct MAC. + If not, try to find a device with the correct MAC and rename it. + 'already_renamed' is used to avoid infinite recursion. + """ - pifrec = db.get_pif_record(pif) - - vlan = pifrec['VLAN_master_of'] - if not vlan or vlan == "OpaqueRef:NULL": - raise Error("PIF is not a VLAN master") - - vlanrec = db.get_vlan_record(vlan) - if not vlanrec: - raise Error("No VLAN record found for PIF") + def read1(name): + file = None + try: + file = open(name, 'r') + return file.readline().rstrip('\n') + finally: + if file != None: + file.close() - return vlanrec['tagged_PIF'] + def get_netdev_mac(device): + try: + return read1("/sys/class/net/%s/address" % device) + except: + # Probably no such device. + return None -def get_vlan_masters_of_pif(pif): - """Returns a list of PIFs which are VLANs on top of the given pif.""" - - pifrec = db.get_pif_record(pif) - vlans = [db.get_vlan_record(v) for v in pifrec['VLAN_slave_of']] - return [v['untagged_PIF'] for v in vlans if v and db.pif_exists(v['untagged_PIF'])] + def get_netdev_tx_queue_len(device): + try: + return int(read1("/sys/class/net/%s/tx_queue_len" % device)) + except: + # Probably no such device. + return None -def interface_deconfigure_commands(interface): - # The use of [!0-9] keeps an interface of 'eth0' from matching - # VLANs attached to eth0 (such as 'eth0.123'), which are distinct - # interfaces. - return ['--del-match=bridge.*.port=%s' % interface, - '--del-match=bonding.%s.[!0-9]*' % interface, - '--del-match=bonding.*.slave=%s' % interface, - '--del-match=vlan.%s.[!0-9]*' % interface, - '--del-match=port.%s.[!0-9]*' % interface, - '--del-match=iface.%s.[!0-9]*' % interface] + def get_netdev_by_mac(mac): + for device in os.listdir("/sys/class/net"): + dev_mac = get_netdev_mac(device) + if (dev_mac and mac.lower() == dev_mac.lower() and + get_netdev_tx_queue_len(device)): + return device + return None -def run_command(command): - log("Running command: " + ' '.join(command)) - if os.spawnl(os.P_WAIT, command[0], *command) != 0: - log("Command failed: " + ' '.join(command)) - return False - return True + def rename_netdev(old_name, new_name): + log("Changing the name of %s to %s" % (old_name, new_name)) + run_command(['/sbin/ifconfig', old_name, 'down']) + if not run_command(['/sbin/ip', 'link', 'set', old_name, 'name', new_name]): + raise Error("Could not rename %s to %s" % (old_name, new_name)) -def rename_netdev(old_name, new_name): - log("Changing the name of %s to %s" % (old_name, new_name)) - run_command(['/sbin/ifconfig', old_name, 'down']) - if not run_command(['/sbin/ip', 'link', 'set', old_name, - 'name', new_name]): - raise Error("Could not rename %s to %s" % (old_name, new_name)) - -# Check whether 'pif' exists and has the correct MAC. -# If not, try to find a device with the correct MAC and rename it. -# 'already_renamed' is used to avoid infinite recursion. -def remap_pif(pif, already_renamed=[]): pifrec = db.get_pif_record(pif) device = pifrec['device'] mac = pifrec['MAC'] # Is there a network device named 'device' at all? - device_exists = interface_exists(device) + device_exists = netdev_exists(device) if device_exists: # Yes. Does it have MAC 'mac'? found_mac = get_netdev_mac(device) @@ -613,88 +750,96 @@ def remap_pif(pif, already_renamed=[]): # Rename 'cur_device' to 'device'. rename_netdev(cur_device, device) -def read_first_line_of_file(name): - file = None - try: - file = open(name, 'r') - return file.readline().rstrip('\n') - finally: - if file != None: - file.close() - -def down_netdev(interface, deconfigure=True): - if not interface_exists(interface): - log("down_netdev: interface %s does not exist, ignoring" % interface) - return - if deconfigure: - # Kill dhclient. - pidfile_name = '/var/run/dhclient-%s.pid' % interface - try: - os.kill(int(read_first_line_of_file(pidfile_name)), signal.SIGTERM) - except: - pass - - # Remove dhclient pidfile. - try: - os.remove(pidfile_name) - except: - pass - - run_command(["/sbin/ifconfig", interface, '0.0.0.0']) +# +# IP Network Devices -- network devices with IP configuration +# - run_command(["/sbin/ifconfig", interface, 'down']) +def pif_ipdev_name(pif): + """Return the ipdev name associated with pif""" + pifrec = db.get_pif_record(pif) + nwrec = db.get_network_record(pifrec['network']) + + if nwrec['bridge']: + # TODO: sanity check that nwrec['bridgeless'] != 'true' + return nwrec['bridge'] + else: + # TODO: sanity check that nwrec['bridgeless'] == 'true' + return pif_netdev_name(pif) -def up_netdev(interface): - run_command(["/sbin/ifconfig", interface, 'up']) +def ifdown(netdev): + """Bring down a network interface""" + if debug_mode(): + return + if not netdev_exists(netdev): + log("ifdown: device %s does not exist, ignoring" % netdev) + return + if not os.path.exists("/etc/sysconfig/network-scripts/ifcfg-%s" % netdev): + log("ifdown: device %s exists but ifcfg %s does not" % (netdev,netdev)) + netdev_down(netdev) + run_command(["/sbin/ifdown", netdev]) + +def ifup(netdev): + """Bring up a network interface""" + if debug_mode(): + return + if not netdev_exists(netdev): + raise Error("ifup: device %s does not exist, ignoring" % netdev) + if not os.path.exists("/etc/sysconfig/network-scripts/ifcfg-%s" % netdev): + raise Error("ifup: device %s exists but ifcfg-%s does not" % (netdev,netdev)) + run_command(["/sbin/ifup", netdev]) + +# +# Bridges +# -def find_distinguished_pifs(pif): - """Returns the PIFs on host that own DNS and the default route. -The peerdns pif will be the one with pif::other-config:peerdns=true, or the mgmt pif if none have this set. -The gateway pif will be the one with pif::other-config:defaultroute=true, or the mgmt pif if none have this set. +def pif_bridge_name(pif): + """Return the bridge name of a pif. -Note: we prune out the bond master pif (if it exists). -This is because when we are called to bring up an interface with a bond master, it is implicit that -we should bring down that master.""" + PIF must not be a VLAN and must be a bridged PIF.""" pifrec = db.get_pif_record(pif) - host = pifrec['host'] - pifs_on_host = [ __pif for __pif in db.get_all_pifs() if - db.get_pif_record(__pif)['host'] == host and - (not __pif in get_bond_masters_of_pif(pif)) ] + if pif_is_vlan(pif): + raise Error("PIF %(uuid)s cannot be a bridge, VLAN is %(VLAN)s" % pifrec) + + nwrec = db.get_network_record(pifrec['network']) - peerdns_pif = None - defaultroute_pif = None - - # loop through all the pifs on this host looking for one with - # other-config:peerdns = true, and one with - # other-config:default-route=true - for __pif in pifs_on_host: - __pifrec = db.get_pif_record(__pif) - __oc = __pifrec['other_config'] - if __oc.has_key('peerdns') and __oc['peerdns'] == 'true': - if peerdns_pif == None: - peerdns_pif = __pif - else: - log('Warning: multiple pifs with "peerdns=true" - choosing %s and ignoring %s' % \ - (db.get_pif_record(peerdns_pif)['device'], __pifrec['device'])) - if __oc.has_key('defaultroute') and __oc['defaultroute'] == 'true': - if defaultroute_pif == None: - defaultroute_pif = __pif - else: - log('Warning: multiple pifs with "defaultroute=true" - choosing %s and ignoring %s' % \ - (db.get_pif_record(defaultroute_pif)['device'], __pifrec['device'])) - - # If no pif is explicitly specified then use the mgmt pif for peerdns/defaultroute - if peerdns_pif == None: - peerdns_pif = management_pif - if defaultroute_pif == None: - defaultroute_pif = management_pif + if nwrec['bridge']: + return nwrec['bridge'] + else: + raise Error("PIF %(uuid)s does not have a bridge name" % pifrec) + +# +# PIF miscellanea +# + +def pif_currently_in_use(pif): + """Determine if a PIF is currently in use. + + A PIF is determined to be currently in use if + - PIF.currently-attached is true + - Any bond master is currently attached + - Any VLAN master is currently attached + """ + rec = db.get_pif_record(pif) + if rec['currently_attached']: + log("configure_datapath: %s is currently attached" % (pif_netdev_name(pif))) + return True + for b in pif_get_bond_masters(pif): + if pif_currently_in_use(b): + log("configure_datapath: %s is in use by BOND master %s" % (pif_netdev_name(pif),pif_netdev_name(b))) + return True + for v in pif_get_vlan_masters(pif): + if pif_currently_in_use(v): + log("configure_datapath: %s is in use by VLAN master %s" % (pif_netdev_name(pif),pif_netdev_name(v))) + return True + return False - return peerdns_pif, defaultroute_pif +# +# +# -def run_ethtool(device, oc): - # Run "ethtool -s" if there are any settings. +def ethtool_settings(oc): settings = [] if oc.has_key('ethtool-speed'): val = oc['ethtool-speed'] @@ -716,10 +861,6 @@ def run_ethtool(device, oc): settings += ['autoneg', 'off'] else: log("Invalid value for ethtool-autoneg = %s. Must be on|true|off|false." % val) - if settings: - run_command(['/sbin/ethtool', '-s', device] + settings) - - # Run "ethtool -K" if there are any offload settings. offload = [] for opt in ("rx", "tx", "sg", "tso", "ufo", "gso"): if oc.has_key("ethtool-" + opt): @@ -730,758 +871,179 @@ def run_ethtool(device, oc): offload += [opt, 'off'] else: log("Invalid value for ethtool-%s = %s. Must be on|true|off|false." % (opt, val)) - if offload: - run_command(['/sbin/ethtool', '-K', device] + offload) + return settings,offload def mtu_setting(oc): if oc.has_key('mtu'): try: int(oc['mtu']) # Check that the value is an integer - return ['mtu', oc['mtu']] + return oc['mtu'] except ValueError, x: - log("Invalid value for mtu = %s" % mtu) - return [] + log("Invalid value for mtu = %s" % oc['mtu']) + return None + +# +# Bonded PIFs +# +def pif_get_bond_masters(pif): + """Returns a list of PIFs which are bond masters of this PIF""" -def configure_local_port(pif): pifrec = db.get_pif_record(pif) - datapath = datapath_name(pif) - ipdev = ipdev_name(pif) - host = pifrec['host'] - nw = pifrec['network'] - nwrec = db.get_network_record(nw) + bso = pifrec['bond_slave_of'] - pif_oc = pifrec['other_config'] - nw_oc = nwrec['other_config'] + # bond-slave-of is currently a single reference but in principle a + # PIF could be a member of several bonds which are not + # concurrently attached. Be robust to this possibility. + if not bso or bso == "OpaqueRef:NULL": + bso = [] + elif not type(bso) == list: + bso = [bso] - # IP (except DHCP) and MTU. - ifconfig_argv = ['/sbin/ifconfig', ipdev, 'up'] - gateway = '' - if pifrec['ip_configuration_mode'] == "DHCP": - pass - elif pifrec['ip_configuration_mode'] == "Static": - ifconfig_argv += [pifrec['IP']] - ifconfig_argv += ['netmask', pifrec['netmask']] - gateway = pifrec['gateway'] - elif pifrec['ip_configuration_mode'] == "None": - # Nothing to do. - pass - else: - raise Error("Unknown IP-configuration-mode %s" % pifrec['ip_configuration_mode']) - ifconfig_argv += mtu_setting(nw_oc) - run_command(ifconfig_argv) - - (peerdns_pif, defaultroute_pif) = find_distinguished_pifs(pif) - - # /etc/resolv.conf - if peerdns_pif == pif: - f = ConfigurationFile('resolv.conf', "/etc") - if pif_oc.has_key('domain'): - f.write("search %s\n" % pif_oc['domain']) - for dns in pifrec['DNS'].split(","): - f.write("nameserver %s\n" % dns) - f.close() - f.apply() - f.commit() + bondrecs = [db.get_bond_record(bond) for bond in bso] + bondrecs = [rec for rec in bondrecs if rec] - # Routing. - if defaultroute_pif == pif and gateway != '': - run_command(['/sbin/ip', 'route', 'replace', 'default', - 'via', gateway, 'dev', ipdev]) - if nw_oc.has_key('static-routes'): - for line in nw_oc['static-routes'].split(','): - network, masklen, gateway = line.split('/') - run_command(['/sbin/ip', 'route', 'add', - '%s/%s' % (network, masklen), 'via', gateway, - 'dev', ipdev]) - - # Ethtool. - run_ethtool(ipdev, nw_oc) - - # DHCP. - if pifrec['ip_configuration_mode'] == "DHCP": - print - print "Determining IP information for %s..." % ipdev, - argv = ['/sbin/dhclient', '-q', - '-lf', '/var/lib/dhclient/dhclient-%s.leases' % ipdev, - '-pf', '/var/run/dhclient-%s.pid' % ipdev, - ipdev] - if run_command(argv): - print 'done.' - else: - print 'failed.' + return [bond['master'] for bond in bondrecs] + +def pif_get_bond_slaves(pif): + """Returns a list of PIFs which make up the given bonded pif.""" -def configure_physdev(pif): pifrec = db.get_pif_record(pif) - device = pifrec['device'] - oc = pifrec['other_config'] - run_command(['/sbin/ifconfig', device, 'up'] + mtu_setting(oc)) - run_ethtool(device, oc) + bmo = pifrec['bond_master_of'] + if len(bmo) > 1: + raise Error("Bond-master-of contains too many elements") -def modify_config(commands): - run_command(['/root/vswitch/bin/ovs-cfg-mod', '-vANY:console:emer', - '-F', '/etc/ovs-vswitchd.conf'] - + commands + ['-c']) - run_command(['/sbin/service', 'vswitch', 'reload']) + if len(bmo) == 0: + return [] -def is_bond_pif(pif): - pifrec = db.get_pif_record(pif) - return len(pifrec['bond_master_of']) != 0 + bondrec = db.get_bond_record(bmo[0]) + if not bondrec: + raise Error("No bond record for bond master PIF") -def configure_bond(pif): - pifrec = db.get_pif_record(pif) - interface = interface_name(pif) - ipdev = ipdev_name(pif) - datapath = datapath_name(pif) - physdev_names = get_physdev_names(pif) + return bondrec['slaves'] - argv = ['--del-match=bonding.%s.[!0-9]*' % interface] - argv += ["--add=bonding.%s.slave=%s" % (interface, slave) - for slave in physdev_names] - argv += ['--add=bonding.%s.fake-iface=true' % interface] +# +# VLAN PIFs +# - if pifrec['MAC'] != "": - argv += ['--add=port.%s.mac=%s' % (interface, pifrec['MAC'])] +def pif_is_vlan(pif): + return db.get_pif_record(pif)['VLAN'] != '-1' - # Bonding options. - bond_options = { - "mode": "balance-slb", - "miimon": "100", - "downdelay": "200", - "updelay": "31000", - "use_carrier": "1", - } - # override defaults with values from other-config whose keys - # being with "bond-" - oc = pifrec['other_config'] - overrides = filter(lambda (key,val): - key.startswith("bond-"), oc.items()) - overrides = map(lambda (key,val): (key[5:], val), overrides) - bond_options.update(overrides) - for (name,val) in bond_options.items(): - argv += ["--add=bonding.%s.%s=%s" % (interface, name, val)] - return argv +def pif_get_vlan_slave(pif): + """Find the PIF which is the VLAN slave of pif. + +Returns the 'physical' PIF underneath the a VLAN PIF @pif.""" -def action_up(pif): pifrec = db.get_pif_record(pif) - bridge = bridge_name(pif) - interface = interface_name(pif) - ipdev = ipdev_name(pif) - datapath = datapath_name(pif) - physdev_pifs = get_physdev_pifs(pif) - physdev_names = get_physdev_names(pif) - vlan_slave = None - if pifrec['VLAN'] != '-1': - vlan_slave = get_vlan_slave_of_pif(pif) - if vlan_slave and is_bond_pif(vlan_slave): - bond_master = vlan_slave - elif is_bond_pif(pif): - bond_master = pif - else: - bond_master = None - if bond_master: - bond_slaves = get_bond_slaves_of_pif(bond_master) - else: - bond_slaves = [] - bond_masters = get_bond_masters_of_pif(pif) + vlan = pifrec['VLAN_master_of'] + if not vlan or vlan == "OpaqueRef:NULL": + raise Error("PIF is not a VLAN master") - # Support "rpm -e vswitch" gracefully by keeping Centos configuration - # files up-to-date, even though we don't use them or need them. - f = configure_pif(pif) - mode = pifrec['ip_configuration_mode'] - if bridge: - log("Configuring %s using %s configuration" % (bridge, mode)) - br = open_network_ifcfg(pif) - configure_network(pif, br) - br.close() - f.attach_child(br) - else: - log("Configuring %s using %s configuration" % (interface, mode)) - configure_network(pif, f) - f.close() - for master in bond_masters: - master_bridge = bridge_name(master) - removed = unconfigure_pif(master) - f.attach_child(removed) - if master_bridge: - removed = open_network_ifcfg(master) - log("Unlinking stale file %s" % removed.path()) - removed.unlink() - f.attach_child(removed) + vlanrec = db.get_vlan_record(vlan) + if not vlanrec: + raise Error("No VLAN record found for PIF") - # /etc/xensource/scripts/vif needs to know where to add VIFs. - if vlan_slave: - if not os.path.exists(vswitch_config_dir): - os.mkdir(vswitch_config_dir) - br = ConfigurationFile("br-%s" % bridge, vswitch_config_dir) - br.write("VLAN_SLAVE=%s\n" % datapath) - br.write("VLAN_VID=%s\n" % pifrec['VLAN']) - br.close() - f.attach_child(br) + return vlanrec['tagged_PIF'] - # Update all configuration files (both ours and Centos's). - f.apply() - f.commit() +def pif_get_vlan_masters(pif): + """Returns a list of PIFs which are VLANs on top of the given pif.""" - # Check the MAC address of each network device and remap if - # necessary to make names match our expectations. - for physdev_pif in physdev_pifs: - remap_pif(physdev_pif) - - # "ifconfig down" the network device and delete its IP address, etc. - down_netdev(ipdev) - for physdev_name in physdev_names: - down_netdev(physdev_name) - - # If we are bringing up a bond, remove IP addresses from the - # slaves (because we are implicitly being asked to take them down). - # - # Conversely, if we are bringing up an interface that has bond - # masters, remove IP addresses from the bond master (because we - # are implicitly being asked to take it down). - for bond_pif in bond_slaves + bond_masters: - run_command(["/sbin/ifconfig", ipdev_name(bond_pif), '0.0.0.0']) - - # Remove all keys related to pif and any bond masters linked to PIF. - del_ports = [ipdev] + physdev_names + bond_masters - if vlan_slave and bond_master: - del_ports += [interface_name(bond_master)] - - # What ports do we need to add to the datapath? - # - # We definitely need the ipdev, and ordinarily we want the - # physical devices too, but for bonds we need the bond as bridge - # port. - add_ports = [ipdev, datapath] - if not bond_master: - add_ports += physdev_names - else: - add_ports += [interface_name(bond_master)] + pifrec = db.get_pif_record(pif) + vlans = [db.get_vlan_record(v) for v in pifrec['VLAN_slave_of']] + return [v['untagged_PIF'] for v in vlans if v and db.pif_exists(v['untagged_PIF'])] - # What ports do we need to delete? - # - # - All the ports that we add, to avoid duplication and to drop - # them from another datapath in case they're misassigned. - # - # - The physical devices, since they will either be in add_ports - # or added to the bonding device (see below). - # - # - The bond masters for pif. (Ordinarily pif shouldn't have any - # bond masters. If it does then interface-reconfigure is - # implicitly being asked to take them down.) - del_ports = add_ports + physdev_names + bond_masters +# +# IP device configuration +# - # What networks does this datapath carry? - # - # - The network corresponding to the datapath's PIF. - # - # - The networks corresponding to any VLANs attached to the - # datapath's PIF. - network_uuids = [] - for nwpif in db.get_pifs_by_record({'device': pifrec['device'], - 'host': pifrec['host']}): - net = db.get_pif_record(nwpif)['network'] - network_uuids += [db.get_network_record(net)['uuid']] - - # Bring up bond slaves early, because ovs-vswitchd initially - # enables or disables bond slaves based on whether carrier is - # detected when they are added, and a network device that is down - # always reports "no carrier". - bond_slave_physdev_pifs = [] - for slave in bond_slaves: - bond_slave_physdev_pifs += get_physdev_pifs(slave) - for slave_physdev_pif in set(bond_slave_physdev_pifs): - configure_physdev(slave_physdev_pif) - - # Now modify the ovs-vswitchd config file. - argv = [] - for port in set(del_ports): - argv += interface_deconfigure_commands(port) - for port in set(add_ports): - argv += ['--add=bridge.%s.port=%s' % (datapath, port)] - if vlan_slave: - argv += ['--add=vlan.%s.tag=%s' % (ipdev, pifrec['VLAN'])] - argv += ['--add=iface.%s.internal=true' % (ipdev)] - - # xapi creates a bridge by the name of the ipdev and requires - # that the IP address will be on it. We need to delete this - # bridge because we need that device to be a member of our - # datapath. - argv += ['--del-match=bridge.%s.[!0-9]*' % ipdev] - - # xapi insists that its attempts to create the bridge succeed, - # so force that to happen. - argv += ['--add=iface.%s.fake-bridge=true' % (ipdev)] +def ipdev_configure_static_routes(interface, oc, f): + """Open a route- file for static routes. + + Opens the static routes configuration file for interface and writes one + line for each route specified in the network's other config "static-routes" value. + E.g. if + interface ( RO): xenbr1 + other-config (MRW): static-routes: 172.16.0.0/15/192.168.0.3,172.18.0.0/16/192.168.0.4;... + + Then route-xenbr1 should be + 172.16.0.0/15 via 192.168.0.3 dev xenbr1 + 172.18.0.0/16 via 192.168.0.4 dev xenbr1 + """ + fname = "route-%s" % interface + if oc.has_key('static-routes'): + # The key is present - extract comma seperates entries + lines = oc['static-routes'].split(',') else: - try: - os.unlink("%s/br-%s" % (vswitch_config_dir, bridge)) - except OSError: - pass - argv += ['--del-match=bridge.%s.xs-network-uuids=*' % datapath] - argv += ['--add=bridge.%s.xs-network-uuids=%s' % (datapath, uuid) - for uuid in set(network_uuids)] - if bond_master: - argv += configure_bond(bond_master) - modify_config(argv) - - # Bring up VLAN slave, plus physical devices other than bond - # slaves (which we brought up earlier). - if vlan_slave: - up_netdev(ipdev_name(vlan_slave)) - for physdev_pif in set(physdev_pifs) - set(bond_slave_physdev_pifs): - configure_physdev(physdev_pif) - - # Configure network device for local port. - configure_local_port(pif) - - # Update /etc/issue (which contains the IP address of the management interface) - os.system("/sbin/update-issue") - - if bond_slaves: - # There seems to be a race somewhere: without this sleep, using - # XenCenter to create a bond that becomes the management interface - # fails with "The underlying connection was closed: A connection that - # was expected to be kept alive was closed by the server." on every - # second or third try, even though /var/log/messages doesn't show - # anything unusual. - # - # The race is probably present even without vswitch, but bringing up a - # bond without vswitch involves a built-in pause of 10 seconds or more - # to wait for the bond to transition from learning to forwarding state. - time.sleep(5) - -def action_down(pif): - rec = db.get_pif_record(pif) - interface = interface_name(pif) - bridge = bridge_name(pif) - ipdev = ipdev_name(pif) - - # Support "rpm -e vswitch" gracefully by keeping Centos configuration - # files up-to-date, even though we don't use them or need them. - f = unconfigure_pif(pif) - if bridge: - br = open_network_ifcfg(pif) - log("Unlinking stale file %s" % br.path()) - br.unlink() - f.attach_child(br) + # The key is not present, i.e. there are no static routes + lines = [] + + child = ConfigurationFile(fname) + child.write("# DO NOT EDIT: This file (%s) was autogenerated by %s\n" % \ + (os.path.basename(child.path()), os.path.basename(sys.argv[0]))) + try: - f.apply() - f.commit() - except Error, e: - log("action_down failed to apply changes: %s" % e.msg) - f.revert() - raise + for l in lines: + network, masklen, gateway = l.split('/') + child.write("%s/%s via %s dev %s\n" % (network, masklen, gateway, interface)) - argv = [] - if rec['VLAN'] != '-1': - # Get rid of the VLAN device itself. - down_netdev(ipdev) - argv += interface_deconfigure_commands(ipdev) + f.attach_child(child) + child.close() - # If the VLAN's slave is attached, stop here. - slave = get_vlan_slave_of_pif(pif) - if db.get_pif_record(slave)['currently_attached']: - log("VLAN slave is currently attached") - modify_config(argv) - return - - # If the VLAN's slave has other VLANs that are attached, stop here. - masters = get_vlan_masters_of_pif(slave) - for m in masters: - if m != pif and db.get_pif_record(m)['currently_attached']: - log("VLAN slave has other master %s" % interface_naem(m)) - modify_config(argv) - return - - # Otherwise, take down the VLAN's slave too. - log("No more masters, bring down vlan slave %s" % interface_name(slave)) - pif = slave - else: - # Stop here if this PIF has attached VLAN masters. - vlan_masters = get_vlan_masters_of_pif(pif) - log("VLAN masters of %s - %s" % (rec['device'], [interface_name(m) for m in vlan_masters])) - for m in vlan_masters: - if db.get_pif_record(m)['currently_attached']: - log("Leaving %s up due to currently attached VLAN master %s" % (interface, interface_name(m))) - return - - # pif is now either a bond or a physical device which needs to be - # brought down. pif might have changed so re-check all its attributes. - rec = db.get_pif_record(pif) - interface = interface_name(pif) - bridge = bridge_name(pif) - ipdev = ipdev_name(pif) + except ValueError, e: + log("Error in other-config['static-routes'] format for network %s: %s" % (interface, e)) +def ipdev_open_ifcfg(pif): + ipdev = pif_ipdev_name(pif) - bond_slaves = get_bond_slaves_of_pif(pif) - log("bond slaves of %s - %s" % (rec['device'], [interface_name(s) for s in bond_slaves])) - for slave in bond_slaves: - slave_interface = interface_name(slave) - log("bring down bond slave %s" % slave_interface) - argv += interface_deconfigure_commands(slave_interface) - down_netdev(slave_interface) + log("Writing network configuration for %s" % ipdev) - argv += interface_deconfigure_commands(ipdev) - down_netdev(ipdev) + f = ConfigurationFile("ifcfg-%s" % ipdev) - argv += ['--del-match', 'bridge.%s.*' % datapath_name(pif)] - argv += ['--del-match', 'bonding.%s.[!0-9]*' % interface] - modify_config(argv) + f.write("# DO NOT EDIT: This file (%s) was autogenerated by %s\n" % \ + (os.path.basename(f.path()), os.path.basename(sys.argv[0]))) + f.write("XEMANAGED=yes\n") + f.write("DEVICE=%s\n" % ipdev) + f.write("ONBOOT=no\n") -def action_rewrite(pif): - # Support "rpm -e vswitch" gracefully by keeping Centos configuration - # files up-to-date, even though we don't use them or need them. - pifrec = db.get_pif_record(pif) - f = configure_pif(pif) - interface = interface_name(pif) - bridge = bridge_name(pif) - mode = pifrec['ip_configuration_mode'] - if bridge: - log("Configuring %s using %s configuration" % (bridge, mode)) - br = open_network_ifcfg(pif) - configure_network(pif, br) - br.close() - f.attach_child(br) - else: - log("Configuring %s using %s configuration" % (interface, mode)) - configure_network(pif, f) - f.close() - try: - f.apply() - f.commit() - except Error, e: - log("failed to apply changes: %s" % e.msg) - f.revert() - raise + return f - # We have no code of our own to run here. - pass +def ipdev_configure_network(pif): + """Write the configuration file for a network. -def main(argv=None): - global output_directory, management_pif - - session = None - pif_uuid = None - pif = None + Writes configuration derived from the network object into the relevant + ifcfg file. The configuration file is passed in, but if the network is + bridgeless it will be ifcfg-, otherwise it will be ifcfg-. - force_interface = None - force_management = False - - if argv is None: - argv = sys.argv + This routine may also write ifcfg files of the networks corresponding to other PIFs + in order to maintain consistency. - try: - try: - shortops = "h" - longops = [ "output-directory=", - "pif=", "pif-uuid=", - "session=", - "force=", - "force-interface=", - "management", - "test-mode", - "device=", "mode=", "ip=", "netmask=", "gateway=", - "help" ] - arglist, args = getopt.gnu_getopt(argv[1:], shortops, longops) - except getopt.GetoptError, msg: - raise Usage(msg) - - force_rewrite_config = {} - - for o,a in arglist: - if o == "--output-directory": - output_directory = a - elif o == "--pif": - pif = a - elif o == "--pif-uuid": - pif_uuid = a - elif o == "--session": - session = a - elif o == "--force-interface" or o == "--force": - force_interface = a - elif o == "--management": - force_management = True - elif o in ["--device", "--mode", "--ip", "--netmask", "--gateway"]: - force_rewrite_config[o[2:]] = a - elif o == "-h" or o == "--help": - print __doc__ % {'command-name': os.path.basename(argv[0])} - return 0 - - if not debug_mode(): - syslog.openlog(os.path.basename(argv[0])) - log("Called as " + str.join(" ", argv)) - if len(args) < 1: - raise Usage("Required option not present") - if len(args) > 1: - raise Usage("Too many arguments") - - action = args[0] - # backwards compatibility - if action == "rewrite-configuration": action = "rewrite" - - if output_directory and ( session or pif ): - raise Usage("--session/--pif cannot be used with --output-directory") - if ( session or pif ) and pif_uuid: - raise Usage("--session/--pif and --pif-uuid are mutually exclusive.") - if ( session and not pif ) or ( not session and pif ): - raise Usage("--session and --pif must be used together.") - if force_interface and ( session or pif or pif_uuid ): - raise Usage("--force is mutually exclusive with --session, --pif and --pif-uuid") - if len(force_rewrite_config) and not (force_interface and action == "rewrite"): - raise Usage("\"--force rewrite\" needed for --device, --mode, --ip, --netmask, and --gateway") - - global db - if force_interface: - log("Force interface %s %s" % (force_interface, action)) - - if action == "rewrite": - action_force_rewrite(force_interface, force_rewrite_config) - else: - db = DatabaseCache(cache_file=dbcache_file) - host = db.extras['host'] - pif = db.get_pif_by_bridge(host, force_interface) - management_pif = db.get_management_pif(host) - - if action == "up": - action_up(pif) - elif action == "down": - action_down(pif) - else: - raise Usage("Unknown action %s" % action) - else: - db = DatabaseCache(session_ref=session) - - if pif_uuid: - pif = db.get_pif_by_uuid(pif_uuid) - - if not pif: - raise Usage("No PIF given") - - if force_management: - # pif is going to be the management pif - management_pif = pif - else: - # pif is not going to be the management pif. - # Search DB cache for pif on same host with management=true - pifrec = db.get_pif_record(pif) - host = pifrec['host'] - management_pif = db.get_management_pif(host) - - log_pif_action(action, pif) - - if not check_allowed(pif): - return 0 - - if action == "up": - action_up(pif) - elif action == "down": - action_down(pif) - elif action == "rewrite": - action_rewrite(pif) - else: - raise Usage("Unknown action %s" % action) - - # Save cache. - pifrec = db.get_pif_record(pif) - db.save(dbcache_file, {'host': pifrec['host']}) - - except Usage, err: - print >>sys.stderr, err.msg - print >>sys.stderr, "For help use --help." - return 2 - except Error, err: - log(err.msg) - return 1 - - return 0 - -# The following code allows interface-reconfigure to keep Centos -# network configuration files up-to-date, even though the vswitch -# never uses them. In turn, that means that "rpm -e vswitch" does not -# have to update any configuration files. - -def configure_ethtool(oc, f): - # Options for "ethtool -s" - settings = None - setting_opts = ["autoneg", "speed", "duplex"] - # Options for "ethtool -K" - offload = None - offload_opts = ["rx", "tx", "sg", "tso", "ufo", "gso"] - - for opt in [opt for opt in setting_opts + offload_opts if oc.has_key("ethtool-" + opt)]: - val = oc["ethtool-" + opt] - - if opt in ["speed"]: - if val in ["10", "100", "1000"]: - val = "speed " + val - else: - log("Invalid value for ethtool-speed = %s. Must be 10|100|1000." % val) - val = None - elif opt in ["duplex"]: - if val in ["half", "full"]: - val = "duplex " + val - else: - log("Invalid value for ethtool-duplex = %s. Must be half|full." % val) - val = None - elif opt in ["autoneg"] + offload_opts: - if val in ["true", "on"]: - val = opt + " on" - elif val in ["false", "off"]: - val = opt + " off" - else: - log("Invalid value for ethtool-%s = %s. Must be on|true|off|false." % (opt, val)) - val = None - - if opt in setting_opts: - if val and settings: - settings = settings + " " + val - else: - settings = val - elif opt in offload_opts: - if val and offload: - offload = offload + " " + val - else: - offload = val - - if settings: - f.write("ETHTOOL_OPTS=\"%s\"\n" % settings) - if offload: - f.write("ETHTOOL_OFFLOAD_OPTS=\"%s\"\n" % offload) - -def configure_mtu(oc, f): - if not oc.has_key('mtu'): - return - - try: - mtu = int(oc['mtu']) - f.write("MTU=%d\n" % mtu) - except ValueError, x: - log("Invalid value for mtu = %s" % mtu) - -def configure_static_routes(interface, oc, f): - """Open a route- file for static routes. - - Opens the static routes configuration file for interface and writes one - line for each route specified in the network's other config "static-routes" value. - E.g. if - interface ( RO): xenbr1 - other-config (MRW): static-routes: 172.16.0.0/15/192.168.0.3,172.18.0.0/16/192.168.0.4;... - - Then route-xenbr1 should be - 172.16.0.0/15 via 192.168.0.3 dev xenbr1 - 172.18.0.0/16 via 192.168.0.4 dev xenbr1 - """ - fname = "route-%s" % interface - if oc.has_key('static-routes'): - # The key is present - extract comma seperates entries - lines = oc['static-routes'].split(',') - else: - # The key is not present, i.e. there are no static routes - lines = [] - - child = ConfigurationFile(fname) - child.write("# DO NOT EDIT: This file (%s) was autogenerated by %s\n" % \ - (os.path.basename(child.path()), os.path.basename(sys.argv[0]))) - - try: - for l in lines: - network, masklen, gateway = l.split('/') - child.write("%s/%s via %s dev %s\n" % (network, masklen, gateway, interface)) - - f.attach_child(child) - child.close() - - except ValueError, e: - log("Error in other-config['static-routes'] format for network %s: %s" % (interface, e)) - -def __open_ifcfg(interface): - """Open a network interface configuration file. - - Opens the configuration file for interface, writes a header and - common options and returns the file object. + params: + pif: Opaque_ref of pif + f : ConfigurationFile(/path/to/ifcfg) to which we append network configuration """ - fname = "ifcfg-%s" % interface - f = ConfigurationFile(fname) - - f.write("# DO NOT EDIT: This file (%s) was autogenerated by %s\n" % \ - (os.path.basename(f.path()), os.path.basename(sys.argv[0]))) - f.write("XEMANAGED=yes\n") - f.write("DEVICE=%s\n" % interface) - f.write("ONBOOT=no\n") - - return f -def open_network_ifcfg(pif): - bridge = bridge_name(pif) - interface = interface_name(pif) - if bridge: - return __open_ifcfg(bridge) - else: - return __open_ifcfg(interface) - - -def open_pif_ifcfg(pif): pifrec = db.get_pif_record(pif) - - log("Configuring %s (%s)" % (interface_name(pif), pifrec['MAC'])) - - f = __open_ifcfg(interface_name(pif)) - - if pifrec.has_key('other_config'): - configure_ethtool(pifrec['other_config'], f) - configure_mtu(pifrec['other_config'], f) - - return f + nwrec = db.get_network_record(pifrec['network']) -def configure_network(pif, f): - """Write the configuration file for a network. + ipdev = pif_ipdev_name(pif) - Writes configuration derived from the network object into the relevant - ifcfg file. The configuration file is passed in, but if the network is - bridgeless it will be ifcfg-, otherwise it will be ifcfg-. + f = ipdev_open_ifcfg(pif) - This routine may also write ifcfg files of the networks corresponding to other PIFs - in order to maintain consistency. + mode = pifrec['ip_configuration_mode'] + log("Configuring %s using %s configuration" % (ipdev, mode)) - params: - pif: Opaque_ref of pif - f : ConfigurationFile(/path/to/ifcfg) to which we append network configuration - """ - - pifrec = db.get_pif_record(pif) - host = pifrec['host'] - nw = pifrec['network'] - nwrec = db.get_network_record(nw) oc = None - bridge = bridge_name(pif) - interface = interface_name(pif) - if bridge: - device = bridge - else: - device = interface - - if nwrec.has_key('other_config'): - configure_ethtool(nwrec['other_config'], f) - configure_mtu(nwrec['other_config'], f) - configure_static_routes(device, nwrec['other_config'], f) - - if pifrec.has_key('other_config'): oc = pifrec['other_config'] - if device == bridge: - f.write("TYPE=Bridge\n") - f.write("DELAY=0\n") - f.write("STP=off\n") - f.write("PIFDEV=%s\n" % interface_name(pif)) - + f.write("TYPE=Ethernet\n") if pifrec['ip_configuration_mode'] == "DHCP": f.write("BOOTPROTO=dhcp\n") f.write("PERSISTENT_DHCLIENT=yes\n") elif pifrec['ip_configuration_mode'] == "Static": - f.write("BOOTPROTO=none\n") + f.write("BOOTPROTO=none\n") f.write("NETMASK=%(netmask)s\n" % pifrec) f.write("IPADDR=%(IP)s\n" % pifrec) f.write("GATEWAY=%(gateway)s\n" % pifrec) @@ -1490,29 +1052,49 @@ def configure_network(pif, f): else: raise Error("Unknown ip-configuration-mode %s" % pifrec['ip_configuration_mode']) + if nwrec.has_key('other_config'): + settings,offload = ethtool_settings(nwrec['other_config']) + if len(settings): + f.write("ETHTOOL_OPTS=\"%s\"\n" % str.join(" ", settings)) + if len(offload): + f.write("ETHTOOL_OFFLOAD_OPTS=\"%s\"\n" % str.join(" ", offload)) + + mtu = mtu_setting(nwrec['other_config']) + if mtu: + f.write("MTU=%s\n" % mtu) + + ipdev_configure_static_routes(ipdev, nwrec['other_config'], f) + if pifrec.has_key('DNS') and pifrec['DNS'] != "": ServerList = pifrec['DNS'].split(",") for i in range(len(ServerList)): f.write("DNS%d=%s\n" % (i+1, ServerList[i])) if oc and oc.has_key('domain'): f.write("DOMAIN='%s'\n" % oc['domain'].replace(',', ' ')) - # We only allow one ifcfg-xenbr* to have PEERDNS=yes and there can be only one GATEWAYDEV in /etc/sysconfig/network. - # The peerdns pif will be the one with pif::other-config:peerdns=true, or the mgmt pif if none have this set. - # The gateway pif will be the one with pif::other-config:defaultroute=true, or the mgmt pif if none have this set. + # We only allow one ifcfg-* to have PEERDNS=yes and there can be + # only one GATEWAYDEV in /etc/sysconfig/network. + # + # The peerdns pif will be the one with + # pif::other-config:peerdns=true, or the mgmt pif if none have + # this set. + # + # The gateway pif will be the one with + # pif::other-config:defaultroute=true, or the mgmt pif if none + # have this set. - # Work out which pif on this host should be the one with PEERDNS=yes and which should be the GATEWAYDEV + # Work out which pif on this host should be the one with + # PEERDNS=yes and which should be the GATEWAYDEV # - # Note: we prune out the bond master pif (if it exists). - # This is because when we are called to bring up an interface with a bond master, it is implicit that - # we should bring down that master. + # Note: we prune out the bond master pif (if it exists). This is + # because when we are called to bring up an interface with a bond + # master, it is implicit that we should bring down that master. pifs_on_host = [ __pif for __pif in db.get_all_pifs() if - db.get_pif_record(__pif)['host'] == host and - (not __pif in get_bond_masters_of_pif(pif)) ] + not __pif in pif_get_bond_masters(pif) ] other_pifs_on_host = [ __pif for __pif in pifs_on_host if __pif != pif ] peerdns_pif = None defaultroute_pif = None - + # loop through all the pifs on this host looking for one with # other-config:peerdns = true, and one with # other-config:default-route=true @@ -1531,16 +1113,18 @@ def configure_network(pif, f): else: log('Warning: multiple pifs with "defaultroute=true" - choosing %s and ignoring %s' % \ (db.get_pif_record(defaultroute_pif)['device'], __pifrec['device'])) - - # If no pif is explicitly specified then use the mgmt pif for peerdns/defaultroute + + # If no pif is explicitly specified then use the mgmt pif for + # peerdns/defaultroute. if peerdns_pif == None: peerdns_pif = management_pif if defaultroute_pif == None: defaultroute_pif = management_pif - - # Update all the other network's ifcfg files and ensure consistency + + # Update all the other network's ifcfg files and ensure + # consistency. for __pif in other_pifs_on_host: - __f = open_network_ifcfg(__pif) + __f = ipdev_open_ifcfg(__pif) peerdns_line_wanted = 'PEERDNS=%s\n' % ((__pif == peerdns_pif) and 'yes' or 'no') lines = __f.readlines() @@ -1557,10 +1141,10 @@ def configure_network(pif, f): else: # There is no need to change this ifcfg file. So don't attach_child. pass - + # ... and for this pif too f.write('PEERDNS=%s\n' % ((pif == peerdns_pif) and 'yes' or 'no')) - + # Update gatewaydev fnetwork = ConfigurationFile("network", "/etc/sysconfig") for line in fnetwork.readlines(): @@ -1568,131 +1152,574 @@ def configure_network(pif, f): continue fnetwork.write(line) if defaultroute_pif: - gatewaydev = bridge_name(defaultroute_pif) + gatewaydev = pif_ipdev_name(defaultroute_pif) if not gatewaydev: - gatewaydev = interface_name(defaultroute_pif) + gatewaydev = pif_netdev_name(defaultroute_pif) fnetwork.write('GATEWAYDEV=%s\n' % gatewaydev) fnetwork.close() f.attach_child(fnetwork) - return - - -def configure_physical_interface(pif): - """Write the configuration for a physical interface. - - Writes the configuration file for the physical interface described by - the pif object. - - Returns the open file handle for the interface configuration file. - """ + return f - pifrec = db.get_pif_record(pif) +# +# Datapath Configuration +# - f = open_pif_ifcfg(pif) +def pif_datapath(pif): + """Return the OpenFlow datapath name associated with pif. +For a non-VLAN PIF, the datapath name is the bridge name. +For a VLAN PIF, the datapath name is the bridge name for the PIF's VLAN slave. +""" + if pif_is_vlan(pif): + return pif_datapath(pif_get_vlan_slave(pif)) - f.write("TYPE=Ethernet\n") - f.write("HWADDR=%(MAC)s\n" % pifrec) + pifrec = db.get_pif_record(pif) + nwrec = db.get_network_record(pifrec['network']) + if not nwrec['bridge']: + raise Error("datapath PIF cannot be bridgeless") + else: + return pif - return f +def datapath_get_physical_pifs(pif): + """Return the PIFs for the physical network device(s) associated with a datapath PIF. +For a bond master PIF, these are the bond slave PIFs. +For a non-VLAN, non-bond master PIF, the PIF is its own physical device PIF. -def configure_bond_interface(pif): - """Write the configuration for a bond interface. +A VLAN PIF cannot be a datapath PIF. +""" + pifrec = db.get_pif_record(pif) - Writes the configuration file for the bond interface described by - the pif object. Handles writing the configuration for the slave - interfaces. + if pif_is_vlan(pif): + raise Error("get-physical-pifs should not get passed a VLAN") + elif len(pifrec['bond_master_of']) != 0: + return pif_get_bond_slaves(pif) + else: + return [pif] - Returns the open file handle for the bond interface configuration - file. - """ +def datapath_deconfigure_physical(netdev): + # The use of [!0-9] keeps an interface of 'eth0' from matching + # VLANs attached to eth0 (such as 'eth0.123'), which are distinct + # interfaces. + return ['--del-match=bridge.*.port=%s' % netdev, + '--del-match=port.%s.[!0-9]*' % netdev, + '--del-match=bonding.*.slave=%s' % netdev, + '--del-match=iface.%s.[!0-9]*' % netdev] +def datapath_configure_bond(pif,slaves): pifrec = db.get_pif_record(pif) - oc = pifrec['other_config'] - f = open_pif_ifcfg(pif) + interface = pif_netdev_name(pif) - if pifrec['MAC'] != "": - f.write("MACADDR=%s\n" % pifrec['MAC']) + argv = ['--del-match=bonding.%s.[!0-9]*' % interface] + argv += ["--add=bonding.%s.slave=%s" % (interface, pif_netdev_name(slave)) + for slave in slaves] + argv += ['--add=bonding.%s.fake-iface=true' % interface] - for slave in get_bond_slaves_of_pif(pif): - s = configure_physical_interface(slave) - s.write("MASTER=%(device)s\n" % pifrec) - s.write("SLAVE=yes\n") - s.close() - f.attach_child(s) + if pifrec['MAC'] != "": + argv += ['--add=port.%s.mac=%s' % (interface, pifrec['MAC'])] - # The bond option defaults - bond_options = { + # Bonding options. + bond_options = { "mode": "balance-slb", "miimon": "100", "downdelay": "200", "updelay": "31000", "use_carrier": "1", } - - # override defaults with values from other-config whose keys being with "bond-" - overrides = filter(lambda (key,val): key.startswith("bond-"), oc.items()) + # override defaults with values from other-config whose keys + # being with "bond-" + oc = pifrec['other_config'] + overrides = filter(lambda (key,val): + key.startswith("bond-"), oc.items()) overrides = map(lambda (key,val): (key[5:], val), overrides) bond_options.update(overrides) - - # write the bond options to ifcfg-bondX - f.write('BONDING_OPTS="') for (name,val) in bond_options.items(): - f.write("%s=%s " % (name,val)) - f.write('"\n') - return f + argv += ["--add=bonding.%s.%s=%s" % (interface, name, val)] + return argv + +def datapath_deconfigure_bond(netdev): + # The use of [!0-9] keeps an interface of 'eth0' from matching + # VLANs attached to eth0 (such as 'eth0.123'), which are distinct + # interfaces. + return ['--del-match=bonding.%s.[!0-9]*' % netdev, + '--del-match=port.%s.[!0-9]*' % netdev] + +def datapath_deconfigure_ipdev(interface): + # The use of [!0-9] keeps an interface of 'eth0' from matching + # VLANs attached to eth0 (such as 'eth0.123'), which are distinct + # interfaces. + return ['--del-match=bridge.*.port=%s' % interface, + '--del-match=port.%s.[!0-9]*' % interface, + '--del-match=iface.%s.[!0-9]*' % interface, + '--del-match=vlan.%s.[!0-9]*' % interface] + +def datapath_modify_config(commands): + if debug_mode(): + log("modifying configuration:") + for c in commands: + log(" %s" % c) -def configure_vlan_interface(pif): - """Write the configuration for a VLAN interface. + rc = run_command(['/root/vswitch/bin/ovs-cfg-mod', '-vANY:console:emer', + '-F', '/etc/ovs-vswitchd.conf'] + + [c for c in commands if c[0] != '#'] + ['-c']) + if not rc: + raise Error("Failed to modify vswitch configuration") + run_command(['/sbin/service', 'vswitch', 'reload']) + return True - Writes the configuration file for the VLAN interface described by - the pif object. Handles writing the configuration for the master - interface if necessary. +# +# Toplevel Datapath Configuration. +# - Returns the open file handle for the VLAN interface configuration - file. +def configure_datapath(pif): + """Bring up the datapath configuration for PIF. + + Should be careful not to glitch existing users of the datapath, e.g. other VLANs etc. + + Should take care of tearing down other PIFs which encompass common physical devices. + + Returns a tuple containing + - A list containing the necessary cfgmod command line arguments + - A list of additional devices which should be brought up after + the configuration is applied. """ - slave = configure_pif(get_vlan_slave_of_pif(pif)) - slave.close() + cfgmod_argv = [] + extra_up_ports = [] + + bridge = pif_bridge_name(pif) + + physical_devices = datapath_get_physical_pifs(pif) + + # Determine additional devices to deconfigure. + # + # Given all physical devices which are part of this PIF we need to + # consider: + # - any additional bond which a physical device is part of. + # - any additional physical devices which are part of an additional bond. + # + # Any of these which are not currently in use should be brought + # down and deconfigured. + extra_down_bonds = [] + extra_down_ports = [] + for p in physical_devices: + for bond in pif_get_bond_masters(p): + if bond == pif: + log("configure_datapath: leaving bond %s up" % pif_netdev_name(bond)) + continue + if bond in extra_down_bonds: + continue + if db.get_pif_record(bond)['currently_attached']: + log("configure_datapath: implicitly tearing down currently-attached bond %s" % pif_netdev_name(bond)) + + extra_down_bonds += [bond] + + for s in pif_get_bond_slaves(bond): + if s in physical_devices: + continue + if s in extra_down_ports: + continue + if pif_currently_in_use(s): + continue + extra_down_ports += [s] + + log("configure_datapath: bridge - %s" % bridge) + log("configure_datapath: physical - %s" % [pif_netdev_name(p) for p in physical_devices]) + log("configure_datapath: extra ports - %s" % [pif_netdev_name(p) for p in extra_down_ports]) + log("configure_datapath: extra bonds - %s" % [pif_netdev_name(p) for p in extra_down_bonds]) + + # Need to fully deconfigure any bridge which any of the: + # - physical devices + # - bond devices + # - sibling devices + # refers to + for brpif in physical_devices + extra_down_ports + extra_down_bonds: + if brpif == pif: + continue + b = pif_bridge_name(brpif) + ifdown(b) + cfgmod_argv += ['# remove bridge %s' % b] + cfgmod_argv += ['--del-match=bridge.%s.*' % b] + + for n in extra_down_ports: + dev = pif_netdev_name(n) + cfgmod_argv += ['# deconfigure sibling physical device %s' % dev] + cfgmod_argv += datapath_deconfigure_physical(dev) + netdev_down(dev) + + for n in extra_down_bonds: + dev = pif_netdev_name(n) + cfgmod_argv += ['# deconfigure bond device %s' % dev] + cfgmod_argv += datapath_deconfigure_bond(dev) + netdev_down(dev) + + for p in physical_devices: + dev = pif_netdev_name(p) + cfgmod_argv += ['# deconfigure physical port %s' % dev] + cfgmod_argv += datapath_deconfigure_physical(dev) + + # Check the MAC address of each network device and remap if + # necessary to make names match our expectations. + for p in physical_devices: + netdev_remap_name(p) + + # Bring up physical devices early, because ovs-vswitchd initially + # enables or disables bond slaves based on whether carrier is + # detected when they are added, and a network device that is down + # always reports "no carrier". + for p in physical_devices: + oc = db.get_pif_record(p)['other_config'] + + dev = pif_netdev_name(p) + + mtu = mtu_setting(oc) + + netdev_up(dev, mtu) + + settings, offload = ethtool_settings(oc) + if len(settings): + run_command(['/sbin/ethtool', '-s', dev] + settings) + if len(offload): + run_command(['/sbin/ethtool', '-K', dev] + offload) + + if len(physical_devices) > 1: + cfgmod_argv += ['# deconfigure bond %s' % pif_netdev_name(pif)] + cfgmod_argv += datapath_deconfigure_bond(pif_netdev_name(pif)) + cfgmod_argv += ['--del-entry=bridge.%s.port=%s' % (bridge,pif_netdev_name(pif))] + cfgmod_argv += ['# configure bond %s' % pif_netdev_name(pif)] + cfgmod_argv += datapath_configure_bond(pif, physical_devices) + cfgmod_argv += ['--add=bridge.%s.port=%s' % (bridge,pif_netdev_name(pif)) ] + extra_up_ports += [pif_netdev_name(pif)] + else: + iface = pif_netdev_name(physical_devices[0]) + cfgmod_argv += ['# add physical device %s' % iface] + cfgmod_argv += ['--add=bridge.%s.port=%s' % (bridge,iface) ] + + return cfgmod_argv,extra_up_ports + +def deconfigure_datapath(pif): + cfgmod_argv = [] + + bridge = pif_bridge_name(pif) + + physical_devices = datapath_get_physical_pifs(pif) + + log("deconfigure_datapath: bridge - %s" % bridge) + log("deconfigure_datapath: physical devices - %s" % [pif_netdev_name(p) for p in physical_devices]) + + for p in physical_devices: + dev = pif_netdev_name(p) + cfgmod_argv += ['# deconfigure physical port %s' % dev] + cfgmod_argv += datapath_deconfigure_physical(dev) + netdev_down(dev) - f = open_pif_ifcfg(pif) - f.write("VLAN=yes\n") - f.attach_child(slave) + if len(physical_devices) > 1: + cfgmod_argv += ['# deconfigure bond %s' % pif_netdev_name(pif)] + cfgmod_argv += datapath_deconfigure_bond(pif_netdev_name(pif)) + + cfgmod_argv += ['# deconfigure bridge %s' % bridge] + cfgmod_argv += ['--del-match=bridge.%s.*' % bridge] - return f + return cfgmod_argv -def configure_pif(pif): - """Write the configuration for a PIF object. +# +# Toplevel actions +# - Writes the configuration file the PIF and all dependent - interfaces (bond slaves and VLAN masters etc). +def action_up(pif): + pifrec = db.get_pif_record(pif) + cfgmod_argv = [] + extra_ports = [] - Returns the open file handle for the interface configuration file. - """ + ipdev = pif_ipdev_name(pif) + dp = pif_datapath(pif) + bridge = pif_bridge_name(dp) + log("action_up: %s on bridge %s" % (ipdev, bridge)) + + ifdown(ipdev) + + if dp: + c,e = configure_datapath(dp) + cfgmod_argv += c + extra_ports += e + + cfgmod_argv += ['# configure xs-network-uuids'] + cfgmod_argv += ['--del-match=bridge.%s.xs-network-uuids=*' % bridge] + + for nwpif in db.get_pifs_by_device(db.get_pif_record(pif)['device']): + rec = db.get_pif_record(nwpif) + + # When state is read from dbcache PIF.currently_attached + # is always assumed to be false... Err on the side of + # listing even detached networks for the time being. + #if nwpif != pif and not rec['currently_attached']: + # log("Network PIF %s not currently attached (%s)" % (rec['uuid'],pifrec['uuid'])) + # continue + nwrec = db.get_network_record(rec['network']) + cfgmod_argv += ['--add=bridge.%s.xs-network-uuids=%s' % (bridge, nwrec['uuid'])] + + cfgmod_argv += ["# deconfigure ipdev %s" % ipdev] + cfgmod_argv += datapath_deconfigure_ipdev(ipdev) + cfgmod_argv += ["# reconfigure ipdev %s" % ipdev] + cfgmod_argv += ['--add=bridge.%s.port=%s' % (bridge, ipdev)] + + f = ipdev_configure_network(pif) + f.close() + + # /etc/xensource/scripts/vif needs to know where to add VIFs. + if pif_is_vlan(pif): + if not bridge: + raise Error("Unbridged VLAN devices not implemented yet") + cfgmod_argv += ['--add=vlan.%s.tag=%s' % (ipdev, pifrec['VLAN'])] + cfgmod_argv += ['--add=iface.%s.internal=true' % (ipdev)] + cfgmod_argv += ['--add=iface.%s.fake-bridge=true' % (ipdev)] + + # Apply updated configuration. + try: + f.apply() + + datapath_modify_config(cfgmod_argv) + + ifup(ipdev) + + for p in extra_ports: + netdev_up(p) + + # Update /etc/issue (which contains the IP address of the management interface) + os.system("/sbin/update-issue") + + f.commit() + except Error, e: + log("failed to apply changes: %s" % e.msg) + f.revert() + raise + +def action_down(pif): pifrec = db.get_pif_record(pif) + cfgmod_argv = [] - if pifrec['VLAN'] != '-1': - f = configure_vlan_interface(pif) - elif len(pifrec['bond_master_of']) != 0: - f = configure_bond_interface(pif) - else: - f = configure_physical_interface(pif) + ipdev = pif_ipdev_name(pif) + dp = pif_datapath(pif) + bridge = pif_bridge_name(dp) + + log("action_down: %s on bridge %s" % (ipdev, bridge)) - bridge = bridge_name(pif) - if bridge: - f.write("BRIDGE=%s\n" % bridge) + ifdown(ipdev) - return f + if dp: + #nw = db.get_pif_record(pif)['network'] + #nwrec = db.get_network_record(nw) + #cfgmod_argv += ['# deconfigure xs-network-uuids'] + #cfgmod_argv += ['--del-entry=bridge.%s.xs-network-uuids=%s' % (bridge,nwrec['uuid'])] -def unconfigure_pif(pif): - """Clear up the files created by configure_pif""" - f = open_pif_ifcfg(pif) - log("Unlinking stale file %s" % f.path()) + log("deconfigure ipdev %s on %s" % (ipdev,bridge)) + cfgmod_argv += ["# deconfigure ipdev %s" % ipdev] + cfgmod_argv += datapath_deconfigure_ipdev(ipdev) + + f = ipdev_open_ifcfg(pif) f.unlink() - return f - + + if pif_is_vlan(pif): + br = ConfigurationFile("br-%s" % bridge, vswitch_state_dir) + br.unlink() + f.attach_child(br) + + # If the VLAN's slave is attached, leave datapath setup. + slave = pif_get_vlan_slave(pif) + if db.get_pif_record(slave)['currently_attached']: + log("action_down: vlan slave is currently attached") + dp = None + + # If the VLAN's slave has other VLANs that are attached, leave datapath setup. + for master in pif_get_vlan_masters(slave): + if master != pif and db.get_pif_record(master)['currently_attached']: + log("action_down: vlan slave has other master: %s" % pif_netdev_name(master)) + dp = None + + # Otherwise, take down the datapath too (fall through) + if dp: + log("action_down: no more masters, bring down slave %s" % bridge) + else: + # Stop here if this PIF has attached VLAN masters. + masters = [db.get_pif_record(m)['VLAN'] for m in pif_get_vlan_masters(pif) if db.get_pif_record(m)['currently_attached']] + if len(masters) > 0: + log("Leaving datapath %s up due to currently attached VLAN masters %s" % (bridge, masters)) + dp = None + + if dp: + cfgmod_argv += deconfigure_datapath(dp) + + try: + f.apply() + + datapath_modify_config(cfgmod_argv) + + f.commit() + except Error, e: + log("action_down failed to apply changes: %s" % e.msg) + f.revert() + raise + +def action_rewrite(pif): + f = ipdev_configure_network(pif) + f.close() + try: + f.apply() + f.commit() + except Error, e: + log("failed to apply changes: %s" % e.msg) + f.revert() + raise + +def action_force_rewrite(bridge, config): + raise Error("Force rewrite is not implemented yet.") + +def main(argv=None): + global output_directory, management_pif + + session = None + pif_uuid = None + pif = None + + force_interface = None + force_management = False + + if argv is None: + argv = sys.argv + + try: + try: + shortops = "h" + longops = [ "output-directory=", + "pif=", "pif-uuid=", + "session=", + "force=", + "force-interface=", + "management", + "device=", "mode=", "ip=", "netmask=", "gateway=", + "help" ] + arglist, args = getopt.gnu_getopt(argv[1:], shortops, longops) + except getopt.GetoptError, msg: + raise Usage(msg) + + force_rewrite_config = {} + + for o,a in arglist: + if o == "--output-directory": + output_directory = a + elif o == "--pif": + pif = a + elif o == "--pif-uuid": + pif_uuid = a + elif o == "--session": + session = a + elif o == "--force-interface" or o == "--force": + force_interface = a + elif o == "--management": + force_management = True + elif o in ["--device", "--mode", "--ip", "--netmask", "--gateway"]: + force_rewrite_config[o[2:]] = a + elif o == "-h" or o == "--help": + print __doc__ % {'command-name': os.path.basename(argv[0])} + return 0 + + if not debug_mode(): + syslog.openlog(os.path.basename(argv[0])) + log("Called as " + str.join(" ", argv)) + if len(args) < 1: + raise Usage("Required option not present") + if len(args) > 1: + raise Usage("Too many arguments") + + action = args[0] + + if not action in ["up", "down", "rewrite", "rewrite-configuration"]: + raise Usage("Unknown action \"%s\"" % action) + + # backwards compatibility + if action == "rewrite-configuration": action = "rewrite" + + if output_directory and ( session or pif ): + raise Usage("--session/--pif cannot be used with --output-directory") + if ( session or pif ) and pif_uuid: + raise Usage("--session/--pif and --pif-uuid are mutually exclusive.") + if ( session and not pif ) or ( not session and pif ): + raise Usage("--session and --pif must be used together.") + if force_interface and ( session or pif or pif_uuid ): + raise Usage("--force is mutually exclusive with --session, --pif and --pif-uuid") + if force_interface == "all" and action != "down": + raise Usage("\"--force all\" only valid for down action") + if len(force_rewrite_config) and not (force_interface and action == "rewrite"): + raise Usage("\"--force rewrite\" needed for --device, --mode, --ip, --netmask, and --gateway") + + global db + if force_interface: + log("Force interface %s %s" % (force_interface, action)) + + if action == "rewrite": + action_force_rewrite(force_interface, force_rewrite_config) + elif action in ["up", "down"]: + if action == "down" and force_interface == "all": + raise Error("Force all interfaces down not implemented yet") + + db = DatabaseCache(cache_file=dbcache_file) + pif = db.get_pif_by_bridge(force_interface) + management_pif = db.get_management_pif() + + if action == "up": + action_up(pif) + elif action == "down": + action_down(pif) + else: + raise Error("Unknown action %s" % action) + else: + db = DatabaseCache(session_ref=session) + + if pif_uuid: + pif = db.get_pif_by_uuid(pif_uuid) + + if action == "rewrite" and not pif: + pass + else: + if not pif: + raise Usage("No PIF given") + + if force_management: + # pif is going to be the management pif + management_pif = pif + else: + # pif is not going to be the management pif. + # Search DB cache for pif on same host with management=true + pifrec = db.get_pif_record(pif) + management_pif = db.get_management_pif() + + log_pif_action(action, pif) + + if not check_allowed(pif): + return 0 + + if action == "up": + action_up(pif) + elif action == "down": + action_down(pif) + elif action == "rewrite": + action_rewrite(pif) + else: + raise Error("Unknown action %s" % action) + + # Save cache. + db.save(dbcache_file) + + except Usage, err: + print >>sys.stderr, err.msg + print >>sys.stderr, "For help use --help." + return 2 + except Error, err: + log(err.msg) + return 1 + + return 0 + if __name__ == "__main__": rc = 1 try: @@ -1705,5 +1732,5 @@ if __name__ == "__main__": if not debug_mode(): syslog.closelog() - + sys.exit(rc) diff --git a/xenserver/vswitch-xen.spec b/xenserver/vswitch-xen.spec index 01f49f6d..c79309d2 100644 --- a/xenserver/vswitch-xen.spec +++ b/xenserver/vswitch-xen.spec @@ -103,7 +103,7 @@ rm -rf \ %clean rm -rf $RPM_BUILD_ROOT -%pre +%post if [ ! -f /etc/xensource-inventory ]; then printf "XenSource inventory not present in /etc/xensource-inventory" exit 1 @@ -143,7 +143,7 @@ EOF fi fi -if test ! -e /etc/ovs-vswitch.dbcache; then +if test ! -e /var/lib/openvswitch/dbcache; then if test "$1" = 1; then printf "Creating xapi database cache... " else @@ -152,34 +152,8 @@ if test ! -e /etc/ovs-vswitch.dbcache; then printf "Re-creating xapi database cache... " fi - source /etc/xensource-inventory - if python - "$INSTALLATION_UUID" </dev/null 2>&1; then :; else cat >>/etc/sysctl.conf <