X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=xenserver%2Fopt_xensource_libexec_interface-reconfigure;h=44a2b1e0c1337a844cfd821d5a72f427153438ec;hb=a20d2466fcd4edc2f2cfbc870c225a3afb96ffd5;hp=6ea369ffbbc3017d743cc839627abf3c1d9aee9b;hpb=064af42167bf4fc9aaea2702d80ce08074b889c0;p=openvswitch diff --git a/xenserver/opt_xensource_libexec_interface-reconfigure b/xenserver/opt_xensource_libexec_interface-reconfigure index 6ea369ff..44a2b1e0 100755 --- a/xenserver/opt_xensource_libexec_interface-reconfigure +++ b/xenserver/opt_xensource_libexec_interface-reconfigure @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright (c) Citrix Systems 2008. All rights reserved. +# Copyright (c) 2008,2009 Citrix Systems, Inc. All rights reserved. # Copyright (c) 2009 Nicira Networks. # """Usage: @@ -63,13 +63,14 @@ import traceback import time import re import pickle +import random output_directory = None db = None management_pif = None -dbcache_file = "/etc/vswitch.dbcache" +dbcache_file = "/etc/ovs-vswitch.dbcache" vswitch_config_dir = "/etc/openvswitch" class Usage(Exception): @@ -248,6 +249,33 @@ def check_allowed(pif): def interface_exists(i): return os.path.exists("/sys/class/net/" + i) +def get_netdev_mac(device): + try: + return read_first_line_of_file("/sys/class/net/%s/address" % device) + except: + # Probably no such device. + return None + +def get_netdev_tx_queue_len(device): + try: + return int(read_first_line_of_file("/sys/class/net/%s/tx_queue_len" + % device)) + except: + # Probably no such device. + return None + +def get_netdev_by_mac(mac): + maybe = None + for device in os.listdir("/sys/class/net"): + dev_mac = get_netdev_mac(device) + if dev_mac and mac.lower() == dev_mac.lower(): + if get_netdev_tx_queue_len(device): + return device + if not maybe: + # Probably a datapath internal port. + maybe = device + return maybe + class DatabaseCache(object): def __init__(self, session_ref=None, cache_file=None): if session_ref and cache_file: @@ -433,24 +461,30 @@ The ipdev name is the same as the bridge name. pifrec = db.get_pif_record(pif) return bridge_name(pif) -def physdev_names(pif): - """Return the name(s) of the physical network device(s) associated with pif. -For a VLAN PIF, the physical devices are the VLAN slave's physical devices. -For a bond master PIF, the physical devices are the bond slaves. -For a non-VLAN, non-bond master PIF, the physical device is the PIF itself. +def get_physdev_pifs(pif): + """Return the PIFs for the physical network device(s) associated with pif. +For a VLAN PIF, this is the VLAN slave's physical device PIF. +For a bond master PIF, these are the bond slave PIFs. +For a non-VLAN, non-bond master PIF, the PIF is its own physical device PIF. """ pifrec = db.get_pif_record(pif) if pifrec['VLAN'] != '-1': - return physdev_names(get_vlan_slave_of_pif(pif)) + return [get_vlan_slave_of_pif(pif)] elif len(pifrec['bond_master_of']) != 0: - physdevs = [] - for slave in get_bond_slaves_of_pif(pif): - physdevs += physdev_names(slave) - return physdevs + return get_bond_slaves_of_pif(pif) else: - return [pifrec['device']] + return [pif] + +def get_physdev_names(pif): + """Return the name(s) of the physical network device(s) associated with pif. +For a VLAN PIF, the physical devices are the VLAN slave's physical devices. +For a bond master PIF, the physical devices are the bond slaves. +For a non-VLAN, non-bond master PIF, the physical device is the PIF itself. +""" + + return [db.get_pif_record(phys)['device'] for phys in get_physdev_pifs(pif)] def log_pif_action(action, pif): pifrec = db.get_pif_record(pif) @@ -543,31 +577,76 @@ def run_command(command): return False return True +def rename_netdev(old_name, new_name): + log("Changing the name of %s to %s" % (old_name, new_name)) + run_command(['/sbin/ifconfig', old_name, 'down']) + if not run_command(['/sbin/ip', 'link', 'set', old_name, + 'name', new_name]): + raise Error("Could not rename %s to %s" % (old_name, new_name)) + +# Check whether 'pif' exists and has the correct MAC. +# If not, try to find a device with the correct MAC and rename it. +# 'already_renamed' is used to avoid infinite recursion. +def remap_pif(pif, already_renamed=[]): + pifrec = db.get_pif_record(pif) + device = pifrec['device'] + mac = pifrec['MAC'] + + # Is there a network device named 'device' at all? + device_exists = interface_exists(device) + if device_exists: + # Yes. Does it have MAC 'mac'? + found_mac = get_netdev_mac(device) + if found_mac and mac.lower() == found_mac.lower(): + # Yes, everything checks out the way we want. Nothing to do. + return + else: + log("No network device %s" % device) + + # What device has MAC 'mac'? + cur_device = get_netdev_by_mac(mac) + if not cur_device: + log("No network device has MAC %s" % mac) + return + + # First rename 'device', if it exists, to get it out of the way + # for 'cur_device' to replace it. + if device_exists: + rename_netdev(device, "dev%d" % random.getrandbits(24)) + + # Rename 'cur_device' to 'device'. + rename_netdev(cur_device, device) + +def read_first_line_of_file(name): + file = None + try: + file = open(name, 'r') + return file.readline().rstrip('\n') + finally: + if file != None: + file.close() + def down_netdev(interface, deconfigure=True): if not interface_exists(interface): log("down_netdev: interface %s does not exist, ignoring" % interface) return - argv = ["/sbin/ifconfig", interface, 'down'] if deconfigure: - argv += ['0.0.0.0'] - # Kill dhclient. pidfile_name = '/var/run/dhclient-%s.pid' % interface - pidfile = None try: - pidfile = open(pidfile_name, 'r') - os.kill(int(pidfile.readline()), signal.SIGTERM) + os.kill(int(read_first_line_of_file(pidfile_name)), signal.SIGTERM) except: pass - if pidfile != None: - pidfile.close() # Remove dhclient pidfile. try: os.remove(pidfile_name) except: pass - run_command(argv) + + run_command(["/sbin/ifconfig", interface, '0.0.0.0']) + + run_command(["/sbin/ifconfig", interface, 'down']) def up_netdev(interface): run_command(["/sbin/ifconfig", interface, 'up']) @@ -618,8 +697,8 @@ we should bring down that master.""" return peerdns_pif, defaultroute_pif -def ethtool_settings(oc): - # Options for "ethtool -s" +def run_ethtool(device, oc): + # Run "ethtool -s" if there are any settings. settings = [] if oc.has_key('ethtool-speed'): val = oc['ethtool-speed'] @@ -641,8 +720,10 @@ def ethtool_settings(oc): settings += ['autoneg', 'off'] else: log("Invalid value for ethtool-autoneg = %s. Must be on|true|off|false." % val) + if settings: + run_command(['/sbin/ethtool', '-s', device] + settings) - # Options for "ethtool -K" + # Run "ethtool -K" if there are any offload settings. offload = [] for opt in ("rx", "tx", "sg", "tso", "ufo", "gso"): if oc.has_key("ethtool-" + opt): @@ -653,10 +734,19 @@ def ethtool_settings(oc): offload += [opt, 'off'] else: log("Invalid value for ethtool-%s = %s. Must be on|true|off|false." % (opt, val)) + if offload: + run_command(['/sbin/ethtool', '-K', device] + offload) - return settings, offload +def mtu_setting(oc): + if oc.has_key('mtu'): + try: + int(oc['mtu']) # Check that the value is an integer + return ['mtu', oc['mtu']] + except ValueError, x: + log("Invalid value for mtu = %s" % mtu) + return [] -def configure_netdev(pif): +def configure_local_port(pif): pifrec = db.get_pif_record(pif) datapath = datapath_name(pif) ipdev = ipdev_name(pif) @@ -665,6 +755,10 @@ def configure_netdev(pif): nw = pifrec['network'] nwrec = db.get_network_record(nw) + pif_oc = pifrec['other_config'] + nw_oc = nwrec['other_config'] + + # IP (except DHCP) and MTU. ifconfig_argv = ['/sbin/ifconfig', ipdev, 'up'] gateway = '' if pifrec['ip_configuration_mode'] == "DHCP": @@ -678,45 +772,37 @@ def configure_netdev(pif): pass else: raise Error("Unknown IP-configuration-mode %s" % pifrec['ip_configuration_mode']) - - oc = {} - if pifrec.has_key('other_config'): - oc = pifrec['other_config'] - if oc.has_key('mtu'): - int(oc['mtu']) # Check that the value is an integer - ifconfig_argv += ['mtu', oc['mtu']] - + ifconfig_argv += mtu_setting(nw_oc) run_command(ifconfig_argv) (peerdns_pif, defaultroute_pif) = find_distinguished_pifs(pif) + # /etc/resolv.conf if peerdns_pif == pif: f = ConfigurationFile('resolv.conf', "/etc") - if oc.has_key('domain'): - f.write("search %s\n" % oc['domain']) + if pif_oc.has_key('domain'): + f.write("search %s\n" % pif_oc['domain']) for dns in pifrec['DNS'].split(","): f.write("nameserver %s\n" % dns) f.close() f.apply() f.commit() + # Routing. if defaultroute_pif == pif and gateway != '': run_command(['/sbin/ip', 'route', 'replace', 'default', 'via', gateway, 'dev', ipdev]) - - if oc.has_key('static-routes'): - for line in oc['static-routes'].split(','): + if nw_oc.has_key('static-routes'): + for line in nw_oc['static-routes'].split(','): network, masklen, gateway = line.split('/') run_command(['/sbin/ip', 'route', 'add', - '%s/%s' % (netmask, masklen), 'via', gateway, + '%s/%s' % (network, masklen), 'via', gateway, 'dev', ipdev]) - settings, offload = ethtool_settings(oc) - if settings: - run_command(['/sbin/ethtool', '-s', ipdev] + settings) - if offload: - run_command(['/sbin/ethtool', '-K', ipdev] + offload) + # Ethtool. + run_ethtool(ipdev, nw_oc) + # DHCP. if pifrec['ip_configuration_mode'] == "DHCP": print print "Determining IP information for %s..." % ipdev, @@ -729,6 +815,14 @@ def configure_netdev(pif): else: print 'failed.' +def configure_physdev(pif): + pifrec = db.get_pif_record(pif) + device = pifrec['device'] + oc = pifrec['other_config'] + + run_command(['/sbin/ifconfig', device, 'up'] + mtu_setting(oc)) + run_ethtool(device, oc) + def modify_config(commands): run_command(['/root/vswitch/bin/ovs-cfg-mod', '-vANY:console:emer', '-F', '/etc/ovs-vswitchd.conf'] @@ -744,11 +838,15 @@ def configure_bond(pif): interface = interface_name(pif) ipdev = ipdev_name(pif) datapath = datapath_name(pif) - physdevs = physdev_names(pif) + physdev_names = get_physdev_names(pif) argv = ['--del-match=bonding.%s.[!0-9]*' % interface] argv += ["--add=bonding.%s.slave=%s" % (interface, slave) - for slave in physdevs] + for slave in physdev_names] + argv += ['--add=bonding.%s.fake-iface=true'] + + if pifrec['MAC'] != "": + argv += ['--add=port.%s.mac=%s' % (interface, pifrec['MAC'])] # Bonding options. bond_options = { @@ -776,7 +874,8 @@ def action_up(pif): interface = interface_name(pif) ipdev = ipdev_name(pif) datapath = datapath_name(pif) - physdevs = physdev_names(pif) + physdev_pifs = get_physdev_pifs(pif) + physdev_names = get_physdev_names(pif) vlan_slave = None if pifrec['VLAN'] != '-1': vlan_slave = get_vlan_slave_of_pif(pif) @@ -786,6 +885,10 @@ def action_up(pif): bond_master = pif else: bond_master = None + if bond_master: + bond_slaves = get_bond_slaves_of_pif(bond_master) + else: + bond_slaves = [] bond_masters = get_bond_masters_of_pif(pif) # Support "rpm -e vswitch" gracefully by keeping Centos configuration @@ -826,13 +929,27 @@ def action_up(pif): f.apply() f.commit() + # Check the MAC address of each network device and remap if + # necessary to make names match our expectations. + for physdev_pif in physdev_pifs: + remap_pif(physdev_pif) + # "ifconfig down" the network device and delete its IP address, etc. down_netdev(ipdev) - for physdev in physdevs: - down_netdev(physdev) + for physdev_name in physdev_names: + down_netdev(physdev_name) + + # If we are bringing up a bond, remove IP addresses from the + # slaves (because we are implicitly being asked to take them down). + # + # Conversely, if we are bringing up an interface that has bond + # masters, remove IP addresses from the bond master (because we + # are implicitly being asked to take it down). + for bond_pif in bond_slaves + bond_masters: + run_command(["/sbin/ifconfig", ipdev_name(bond_pif), '0.0.0.0']) # Remove all keys related to pif and any bond masters linked to PIF. - del_ports = [ipdev] + physdevs + bond_masters + del_ports = [ipdev] + physdev_names + bond_masters if vlan_slave and bond_master: del_ports += [interface_name(bond_master)] @@ -843,7 +960,7 @@ def action_up(pif): # port. add_ports = [ipdev, datapath] if not bond_master: - add_ports += physdevs + add_ports += physdev_names else: add_ports += [interface_name(bond_master)] @@ -858,7 +975,7 @@ def action_up(pif): # - The bond masters for pif. (Ordinarily pif shouldn't have any # bond masters. If it does then interface-reconfigure is # implicitly being asked to take them down.) - del_ports = add_ports + physdevs + bond_masters + del_ports = add_ports + physdev_names + bond_masters # What networks does this datapath carry? # @@ -872,6 +989,16 @@ def action_up(pif): net = db.get_pif_record(nwpif)['network'] network_uuids += [db.get_network_record(net)['uuid']] + # Bring up bond slaves early, because ovs-vswitchd initially + # enables or disables bond slaves based on whether carrier is + # detected when they are added, and a network device that is down + # always reports "no carrier". + bond_slave_physdev_pifs = [] + for slave in bond_slaves: + bond_slave_physdev_pifs += get_physdev_pifs(slave) + for slave_physdev_pif in set(bond_slave_physdev_pifs): + configure_physdev(slave_physdev_pif) + # Now modify the ovs-vswitchd config file. argv = [] for port in set(del_ports): @@ -903,17 +1030,31 @@ def action_up(pif): argv += configure_bond(bond_master) modify_config(argv) - # Configure network devices. - configure_netdev(pif) - - # Bring up VLAN slave and bond slaves. + # Bring up VLAN slave, plus physical devices other than bond + # slaves (which we brought up earlier). if vlan_slave: up_netdev(ipdev_name(vlan_slave)) - for physdev in physdevs: - up_netdev(physdev) + for physdev_pif in set(physdev_pifs) - set(bond_slave_physdev_pifs): + configure_physdev(physdev_pif) + + # Configure network device for local port. + configure_local_port(pif) # Update /etc/issue (which contains the IP address of the management interface) os.system("/sbin/update-issue") + + if bond_slaves: + # There seems to be a race somewhere: without this sleep, using + # XenCenter to create a bond that becomes the management interface + # fails with "The underlying connection was closed: A connection that + # was expected to be kept alive was closed by the server." on every + # second or third try, even though /var/log/messages doesn't show + # anything unusual. + # + # The race is probably present even without vswitch, but bringing up a + # bond without vswitch involves a built-in pause of 10 seconds or more + # to wait for the bond to transition from learning to forwarding state. + time.sleep(5) def action_down(pif): rec = db.get_pif_record(pif)