From 404c169247b1c3ac2ebad887f0421478a6cef924 Mon Sep 17 00:00:00 2001 From: Andrew Evans Date: Thu, 10 Feb 2011 16:50:49 -0800 Subject: [PATCH] xenserver: Merge upstream changes and drop pre-5.6.100 support. Citrix have made modifications to the various interface configuration scripts that OVS replaces, so at present those modifications are lost when OVS is installed on e.g. XenServer 5.6.100. This commit applies those changes while preserving OVS-specific modifications that have been made in the interim. One major change introduced by these updates is the removal of support for XenServer releases prior to 5.6.100. This commit also updates the OVS-supplied xen-bugtool in similar fashion. We will soon remove xen-bugtool from OVS altogether and move the added OVS functionality to an XML extension as described in the Supplemental Pack DDK guide. --- INSTALL.Linux | 2 +- INSTALL.XenServer | 5 +- tests/interface-reconfigure.at | 14 ++ vswitchd/vswitch.xml | 2 +- xenserver/README | 6 +- xenserver/etc_xensource_scripts_vif | 38 +---- ..._xensource_libexec_InterfaceReconfigure.py | 40 ++--- ...urce_libexec_InterfaceReconfigureBridge.py | 7 +- ...rce_libexec_InterfaceReconfigureVswitch.py | 2 +- ...pt_xensource_libexec_interface-reconfigure | 2 + xenserver/usr_sbin_xen-bugtool | 152 ++++++++++++------ 11 files changed, 144 insertions(+), 126 deletions(-) diff --git a/INSTALL.Linux b/INSTALL.Linux index 9d69bd86..8ba73ed3 100644 --- a/INSTALL.Linux +++ b/INSTALL.Linux @@ -3,7 +3,7 @@ This document describes how to build and install Open vSwitch on a generic Linux host. If you want to install Open vSwitch on a Citrix -XenServer version 5.5.0, see INSTALL.XenServer instead. +XenServer, see INSTALL.XenServer instead. This version of Open vSwitch may be built manually with "configure" and "make", as described below. You may also build Debian packages by diff --git a/INSTALL.XenServer b/INSTALL.XenServer index 33b7f8ba..e7be494c 100644 --- a/INSTALL.XenServer +++ b/INSTALL.XenServer @@ -5,8 +5,7 @@ This document describes how to build and install Open vSwitch on a Citrix XenServer host. If you want to install Open vSwitch on a generic Linux host, see INSTALL.Linux instead. -These instructions have been tested with XenServer versions 5.5.0, -5.5.0-24648p (Update 1), 5.5.0-25727p (Update 2), and 5.6.0. +These instructions have been tested with XenServer 5.6 FP1. Building Open vSwitch for XenServer ----------------------------------- @@ -38,7 +37,7 @@ where: name of the Open vSwitch tarball, e.g. 0.90.0. is the version number of the Xen kernel, - e.g. 2.6.18-128.1.1.el5.xs5.5.0.487.1006xen. This version number + e.g. 2.6.32.12-0.7.1.xs5.6.100.307.170586xen. This version number appears as the name of a directory in /lib/modules inside the VM. It always ends in "xen". diff --git a/tests/interface-reconfigure.at b/tests/interface-reconfigure.at index bb7eff76..f47ca98f 100644 --- a/tests/interface-reconfigure.at +++ b/tests/interface-reconfigure.at @@ -73,6 +73,7 @@ EOF + False @@ -120,6 +121,7 @@ EOF + False @@ -167,6 +169,7 @@ EOF + False @@ -214,6 +217,7 @@ EOF + False @@ -265,6 +269,7 @@ EOF OpaqueRef:e623e1d6-cd02-be8d-820d-49d65c710297 + False @@ -312,6 +317,7 @@ EOF + True @@ -359,6 +365,7 @@ EOF + False @@ -410,6 +417,7 @@ EOF OpaqueRef:4c0eb823-4d96-da1d-e75f-411b85badb0c + False @@ -656,6 +664,7 @@ Applying changes to /etc/sysconfig/network-scripts/route-xenbr2 configuration Applying changes to /etc/sysconfig/network configuration Applying changes to /etc/sysconfig/network-scripts/ifcfg-xenbr2 configuration /sbin/ifconfig eth2 up mtu 1500 +/sbin/ethtool -K eth2 gro off lro off /usr/bin/ovs-vsctl --timeout=20 --with-iface --if-exists del-port eth2 --may-exist add-br xenbr2 @@ -720,6 +729,7 @@ configure_datapath: extra bonds - [] Applying changes to /etc/sysconfig/network-scripts/route-xapi3 configuration Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi3 configuration /sbin/ifconfig eth3 up mtu 1500 +/sbin/ethtool -K eth3 gro off lro off /usr/bin/ovs-vsctl --timeout=20 --with-iface --if-exists del-port eth3 --may-exist add-br xenbr3 @@ -800,7 +810,9 @@ netdev: down: device xenbr1 does not exist, ignoring Applying changes to /etc/sysconfig/network-scripts/route-xapi1 configuration Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi1 configuration /sbin/ifconfig eth0 up mtu 1500 +/sbin/ethtool -K eth0 gro off lro off /sbin/ifconfig eth1 up mtu 1500 +/sbin/ethtool -K eth1 gro off lro off /usr/bin/ovs-vsctl --timeout=20 --if-exists del-br xenbr0 --if-exists del-br xenbr1 @@ -882,7 +894,9 @@ netdev: down: device xenbr1 does not exist, ignoring Applying changes to /etc/sysconfig/network-scripts/route-xapi2 configuration Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi2 configuration /sbin/ifconfig eth0 up mtu 1500 +/sbin/ethtool -K eth0 gro off lro off /sbin/ifconfig eth1 up mtu 1500 +/sbin/ethtool -K eth1 gro off lro off /usr/bin/ovs-vsctl --timeout=20 --if-exists del-br xenbr0 --if-exists del-br xenbr1 diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml index f4515f4f..c80e8cdb 100644 --- a/vswitchd/vswitch.xml +++ b/vswitchd/vswitch.xml @@ -244,7 +244,7 @@

The version of the system identified by , - e.g. 5.5.0-24648p on XenServer 5.5.0 build 24648. + e.g. 5.6.100-39265p on XenServer 5.6.100 build 39265.

System integrators are responsible for choosing and setting an diff --git a/xenserver/README b/xenserver/README index 7da3ac26..3110aabc 100644 --- a/xenserver/README +++ b/xenserver/README @@ -79,8 +79,8 @@ files are: uuid.py This is uuid.py from Python 2.5. It is installed into the - Open vSwitch RPM because XenServer 5.5 and 5.6 use Python 2.4, - which do not have uuid.py. + Open vSwitch RPM because XenServer uses Python 2.4, which does + not have uuid.py. To install, build the Open vSwitch RPM with a command like this: @@ -94,7 +94,7 @@ question and reboot them. (The openvswitch-dbg package that is also produced need not be installed, but it is harmless to do so.) ---------------------------------------------------------------------- -Copyright (C) 2009, 2010 Nicira Networks, Inc. +Copyright (C) 2009, 2010, 2011 Nicira Networks, Inc. Copying and distribution of this file, with or without modification, are permitted in any medium without royalty provided the copyright diff --git a/xenserver/etc_xensource_scripts_vif b/xenserver/etc_xensource_scripts_vif index 88006e2c..5675cdfe 100755 --- a/xenserver/etc_xensource_scripts_vif +++ b/xenserver/etc_xensource_scripts_vif @@ -25,16 +25,6 @@ IP="/sbin/ip" vsctl="/usr/bin/ovs-vsctl" -# XAPI before build 29381 (approximately) did not provide some of the -# data in XenStore that we rely on. -. /etc/xensource-inventory -if test "$PRODUCT_VERSION" = "5.5.0" || test "${BUILD_NUMBER%[a-z]}" -le 26131 -then - xs550=true -else - xs550=false -fi - handle_promiscuous() { local arg=$(xenstore-read "${PRIVATE}/other-config/promiscuous" 2>/dev/null) @@ -96,43 +86,19 @@ handle_vswitch_vif_details() fi local vif_uuid=$(xenstore-read "${PRIVATE}/vif-uuid" 2>/dev/null) - if $xs550 && [ -z "${vif_uuid}" ] && [ -n "${vm_uuid}" ]; then - vif_uuid=$(xe vif-list --minimal vm-uuid="${vm_uuid}" device=$DEVID) - fi if [ -n "${vif_uuid}" ] ; then - set_vif_external_id "xs-vif-uuid" "${vif_uuid}" + set_vif_external_id "xs-vif-uuid" "${vif_uuid}" fi local vif_details= local net_uuid=$(xenstore-read "${PRIVATE}/network-uuid" 2>/dev/null) - if $xs550 && [ -z "${net_uuid}" ] && [ -n "${vif_uuid}" ]; then - net_uuid=$(xe vif-param-get uuid="${vif_uuid}" param-name=network-uuid) - fi if [ -n "${net_uuid}" ] ; then - set_vif_external_id "xs-network-uuid" "${net_uuid}" + set_vif_external_id "xs-network-uuid" "${net_uuid}" fi - local address=$(xenstore-read "/local/domain/$DOMID/device/vif/$DEVID/mac" 2>/dev/null) if [ -n "${address}" ] ; then set_vif_external_id "attached-mac" "${address}" fi - - if $xs550; then - # vNetManager needs to know the network UUID(s) associated with each - # datapath. Normally interface-reconfigure adds them, but XAPI does - # not use interface-reconfigure for internal networks. Instead, XAPI - # calls the addbr ioctl internally, so we have to do it here instead - # for internal networks. This is only acceptable because xapi is lazy - # about creating internal networks: it only creates one just before it - # adds the first vif to it. There may still be a brief delay between - # the initial ovs-vswitchd connection to vNetManager and setting this - # configuration variable, but vNetManager can tolerate that. - local bridge=$1 - if [ -n "${net_uuid}" ] ; then - logger -t scripts-vif "${bridge} xs-network-uuids ${net_uuid}" - echo "-- br-set-external-id $bridge xs-network-uuids ${net_uuid}" - fi - fi } add_to_bridge() diff --git a/xenserver/opt_xensource_libexec_InterfaceReconfigure.py b/xenserver/opt_xensource_libexec_InterfaceReconfigure.py index 68f7204c..a9bbf07e 100644 --- a/xenserver/opt_xensource_libexec_InterfaceReconfigure.py +++ b/xenserver/opt_xensource_libexec_InterfaceReconfigure.py @@ -279,7 +279,7 @@ _BOND_XML_TAG = "bond" _NETWORK_XML_TAG = "network" _POOL_XML_TAG = "pool" -_ETHTOOL_OTHERCONFIG_ATTRS = ['ethtool-%s' % x for x in 'autoneg', 'speed', 'duplex', 'rx', 'tx', 'sg', 'tso', 'ufo', 'gso' ] +_ETHTOOL_OTHERCONFIG_ATTRS = ['ethtool-%s' % x for x in 'autoneg', 'speed', 'duplex', 'rx', 'tx', 'sg', 'tso', 'ufo', 'gso', 'gro', 'lro' ] _PIF_OTHERCONFIG_ATTRS = [ 'domain', 'peerdns', 'defaultroute', 'mtu', 'static-routes' ] + \ [ 'bond-%s' % x for x in 'mode', 'miimon', 'downdelay', @@ -391,12 +391,7 @@ class DatabaseCache(object): continue self.__pifs[p] = {} for f in _PIF_ATTRS: - if f in [ "tunnel_access_PIF_of", "tunnel_transport_PIF_of" ] and f not in rec: - # XenServer 5.5 network records did not have - # these fields, so allow them to be missing. - pass - else: - self.__pifs[p][f] = rec[f] + self.__pifs[p][f] = rec[f] self.__pifs[p]['other_config'] = {} for f in _PIF_OTHERCONFIG_ATTRS: if not rec['other_config'].has_key(f): continue @@ -404,8 +399,7 @@ class DatabaseCache(object): def __get_vlan_records_from_xapi(self, session): self.__vlans = {} - for v in session.xenapi.VLAN.get_all(): - rec = session.xenapi.VLAN.get_record(v) + for (v,rec) in session.xenapi.VLAN.get_all_records().items(): if not self.__pif_on_host(rec['untagged_PIF']): continue self.__vlans[v] = {} @@ -424,8 +418,7 @@ class DatabaseCache(object): def __get_bond_records_from_xapi(self, session): self.__bonds = {} - for b in session.xenapi.Bond.get_all(): - rec = session.xenapi.Bond.get_record(b) + for (b,rec) in session.xenapi.Bond.get_all_records().items(): if not self.__pif_on_host(rec['master']): continue self.__bonds[b] = {} @@ -434,8 +427,7 @@ class DatabaseCache(object): def __get_network_records_from_xapi(self, session): self.__networks = {} - for n in session.xenapi.network.get_all(): - rec = session.xenapi.network.get_record(n) + for (n,rec) in session.xenapi.network.get_all_records().items(): self.__networks[n] = {} for f in _NETWORK_ATTRS: if f == "PIFs": @@ -512,13 +504,7 @@ class DatabaseCache(object): self.__get_pif_records_from_xapi(session, host) - try: - self.__get_tunnel_records_from_xapi(session) - except XenAPI.Failure, e: - error,details = e.details - if error == "MESSAGE_METHOD_UNKNOWN" and details == "tunnel.get_all": - pass - + self.__get_tunnel_records_from_xapi(session) self.__get_pool_records_from_xapi(session) self.__get_vlan_records_from_xapi(session) self.__get_bond_records_from_xapi(session) @@ -677,8 +663,9 @@ class DatabaseCache(object): # # # +PIF_OTHERCONFIG_DEFAULTS = {'gro': 'off', 'lro': 'off'} -def ethtool_settings(oc): +def ethtool_settings(oc, defaults = {}): settings = [] if oc.has_key('ethtool-speed'): val = oc['ethtool-speed'] @@ -688,8 +675,8 @@ def ethtool_settings(oc): log("Invalid value for ethtool-speed = %s. Must be 10|100|1000." % val) if oc.has_key('ethtool-duplex'): val = oc['ethtool-duplex'] - if val in ["10", "100", "1000"]: - settings += ['duplex', 'val'] + if val in ["half", "full"]: + settings += ['duplex', val] else: log("Invalid value for ethtool-duplex = %s. Must be half|full." % val) if oc.has_key('ethtool-autoneg'): @@ -701,7 +688,7 @@ def ethtool_settings(oc): else: log("Invalid value for ethtool-autoneg = %s. Must be on|true|off|false." % val) offload = [] - for opt in ("rx", "tx", "sg", "tso", "ufo", "gso"): + for opt in ("rx", "tx", "sg", "tso", "ufo", "gso", "gro", "lro"): if oc.has_key("ethtool-" + opt): val = oc["ethtool-" + opt] if val in ["true", "on"]: @@ -710,6 +697,8 @@ def ethtool_settings(oc): offload += [opt, 'off'] else: log("Invalid value for ethtool-%s = %s. Must be on|true|off|false." % (opt, val)) + elif opt in defaults: + offload += [opt, defaults[opt]] return settings,offload # By default the MTU is taken from the Network.MTU setting for VIF, @@ -882,8 +871,7 @@ def pif_get_vlan_masters(pif): # Tunnel PIFs # def pif_is_tunnel(pif): - rec = db().get_pif_record(pif) - return rec.has_key('tunnel_access_PIF_of') and len(rec['tunnel_access_PIF_of']) > 0 + return len(db().get_pif_record(pif)['tunnel_access_PIF_of']) > 0 # # Datapath base class diff --git a/xenserver/opt_xensource_libexec_InterfaceReconfigureBridge.py b/xenserver/opt_xensource_libexec_InterfaceReconfigureBridge.py index 8803122d..a93e4386 100644 --- a/xenserver/opt_xensource_libexec_InterfaceReconfigureBridge.py +++ b/xenserver/opt_xensource_libexec_InterfaceReconfigureBridge.py @@ -274,7 +274,8 @@ def _configure_physical_interface(pif): f.write("TYPE=Ethernet\n") f.write("HWADDR=%(MAC)s\n" % pifrec) - settings,offload = ethtool_settings(pifrec['other_config']) + settings,offload = ethtool_settings(pifrec['other_config'], + PIF_OTHERCONFIG_DEFAULTS) if len(settings): f.write("ETHTOOL_OPTS=\"%s\"\n" % str.join(" ", settings)) if len(offload): @@ -338,7 +339,7 @@ def _configure_bond_interface(pif): if len(offload): f.write("ETHTOOL_OFFLOAD_OPTS=\"%s\"\n" % str.join(" ", offload)) - mtu = mtu_setting(pifrec['network'], "VLAN-PIF", pifrec['other_config']) + mtu = mtu_setting(pifrec['network'], "Bond-PIF", pifrec['other_config']) if mtu: f.write("MTU=%s\n" % mtu) @@ -389,7 +390,7 @@ def _configure_vlan_interface(pif): if len(offload): f.write("ETHTOOL_OFFLOAD_OPTS=\"%s\"\n" % str.join(" ", offload)) - mtu = mtu_setting(pifrec['network'], "Bond-PIF", pifrec['other_config']) + mtu = mtu_setting(pifrec['network'], "VLAN-PIF", pifrec['other_config']) if mtu: f.write("MTU=%s\n" % mtu) diff --git a/xenserver/opt_xensource_libexec_InterfaceReconfigureVswitch.py b/xenserver/opt_xensource_libexec_InterfaceReconfigureVswitch.py index 33beb76c..928e6cef 100644 --- a/xenserver/opt_xensource_libexec_InterfaceReconfigureVswitch.py +++ b/xenserver/opt_xensource_libexec_InterfaceReconfigureVswitch.py @@ -529,7 +529,7 @@ class DatapathVswitch(Datapath): netdev_up(dev, mtu) - settings, offload = ethtool_settings(oc) + settings, offload = ethtool_settings(oc, PIF_OTHERCONFIG_DEFAULTS) if len(settings): run_command(['/sbin/ethtool', '-s', dev] + settings) if len(offload): diff --git a/xenserver/opt_xensource_libexec_interface-reconfigure b/xenserver/opt_xensource_libexec_interface-reconfigure index f1c1336c..ff4b9766 100755 --- a/xenserver/opt_xensource_libexec_interface-reconfigure +++ b/xenserver/opt_xensource_libexec_interface-reconfigure @@ -527,6 +527,8 @@ def action_force_rewrite(bridge, config): f.write('\t\t\n') f.write('\t\tOpaqueRef:NULL\n') f.write('\t\t-1\n') + f.write('\t\t\n') + f.write('\t\t\n') f.write('\t\t%s\n' % interface) f.write('\t\t%s\n' % mac) f.write('\t\t\n') diff --git a/xenserver/usr_sbin_xen-bugtool b/xenserver/usr_sbin_xen-bugtool index af88c74a..aeffed40 100755 --- a/xenserver/usr_sbin_xen-bugtool +++ b/xenserver/usr_sbin_xen-bugtool @@ -79,7 +79,8 @@ FSTAB = '/etc/fstab' PROC_MOUNTS = '/proc/mounts' ISCSI_CONF = '/etc/iscsi/iscsid.conf' ISCSI_INITIATOR = '/etc/iscsi/initiatorname.iscsi' -LVM_CACHE = '/etc/lvm/.cache' +LVM_CACHE = '/etc/lvm/cache/.cache' +LVM_CONFIG = '/etc/lvm/lvm.conf' PROC_CPUINFO = '/proc/cpuinfo' PROC_MEMINFO = '/proc/meminfo' PROC_IOPORTS = '/proc/ioports' @@ -109,6 +110,7 @@ IFCFG_RE = re.compile(r'^.*/ifcfg-.*') ROUTE_RE = re.compile(r'^.*/route-.*') NETWORK_DBCACHE = '/var/xapi/network.dbcache' RESOLV_CONF = '/etc/resolv.conf' +MPP_CONF = '/etc/mpp.conf' MULTIPATH_CONF = '/etc/multipath.conf' NSSWITCH_CONF = '/etc/nsswitch.conf' NTP_CONF = '/etc/ntp.conf' @@ -152,11 +154,11 @@ INSTALLED_REPOS_DIR = '/etc/xensource/installed-repos' PATCH_APPLIED_DIR = '/var/patch/applied' XENSERVER_LOGS = \ [ VAR_LOG_DIR + x for x in - ['xensource.log', 'audit.log', 'xenstored-access.log', 'SMlog', 'xen/xenstored-trace.log', + ['xensource.log', 'audit.log', 'xenstored-access.log', 'SMlog', 'VMPRlog', 'xen/xenstored-trace.log', 'xen/xen-hotplug.log', 'xen/domain-builder-ng.log', 'squeezed.log', 'openvswitch/ovs-brcompatd.log', 'openvswitch/ovs-vswitchd.log', 'openvswitch/ovsdb-server.log' ] + [ f % n for n in range(1, 20) \ - for f in ['xensource.log.%d', 'xensource.log.%d.gz','SMlog.%d', 'SMlog.%d.gz', + for f in ['xensource.log.%d', 'xensource.log.%d.gz','SMlog.%d', 'SMlog.%d.gz', 'VMPRlog.%d', 'VMPRlog.%d.gz', 'audit.log.%d', 'audit.log.%d.gz', 'xenstored-access.log.%d', 'xenstored-access.log.%d.gz', \ 'xen/xenstored-access.log.%d', 'xen/xenstored-access.log.%d.gz', 'squeezed.log.%d', \ 'openvswitch/ovs-brcompatd.log.%d', 'openvswitch/ovs-brcompatd.log.%d.gz', \ @@ -211,6 +213,7 @@ LVS = '/usr/sbin/lvs' LVDISPLAY = '/usr/sbin/lvdisplay' MD5SUM = '/usr/bin/md5sum' MODINFO = '/sbin/modinfo' +MPPUTIL = '/usr/sbin/mppUtil' MULTIPATHD = '/sbin/multipathd' NETSTAT = '/bin/netstat' OVS_DPCTL = '/usr/bin/ovs-dpctl' @@ -322,6 +325,8 @@ dbg = False def cap(key, pii=PII_MAYBE, min_size=-1, max_size=-1, min_time=-1, max_time=-1, mime=MIME_TEXT, checked=True, hidden=False): + if os.getenv('XEN_RT') and max_time > 0: + max_time *= 5 caps[key] = (key, pii, min_size, max_size, min_time, max_time, mime, checked, hidden) cap_sizes[key] = 0 @@ -331,11 +336,11 @@ cap(CAP_BLOBS, PII_NO, max_size=5*MB) cap(CAP_BOOT_LOADER, PII_NO, max_size=3*KB, max_time=5) cap(CAP_CVSM, PII_NO, max_size=3*MB, - max_time=60) -cap(CAP_DISK_INFO, PII_MAYBE, max_size=25*KB, + max_time=120) +cap(CAP_DISK_INFO, PII_MAYBE, max_size=50*KB, max_time=20) cap(CAP_FIRSTBOOT, PII_YES, min_size=60*KB, max_size=80*KB) -cap(CAP_HARDWARE_INFO, PII_MAYBE, max_size=30*KB, +cap(CAP_HARDWARE_INFO, PII_MAYBE, max_size=50*KB, max_time=20) cap(CAP_HDPARM_T, PII_NO, min_size=0, max_size=5*KB, min_time=20, max_time=90, checked=False, hidden=True) @@ -345,13 +350,13 @@ cap(CAP_HOST_CRASHDUMP_LOGS, PII_NO) cap(CAP_KERNEL_INFO, PII_MAYBE, max_size=120*KB, max_time=5) cap(CAP_LOSETUP_A, PII_MAYBE, max_size=KB, max_time=5) -cap(CAP_MULTIPATH, PII_MAYBE, max_size=10*KB, +cap(CAP_MULTIPATH, PII_MAYBE, max_size=20*KB, max_time=10) cap(CAP_NETWORK_CONFIG, PII_IF_CUSTOMIZED, - min_size=0, max_size=20*KB) + min_size=0, max_size=40*KB) cap(CAP_NETWORK_STATUS, PII_YES, max_size=19*KB, max_time=30) -cap(CAP_PAM, PII_NO, max_size=30*KB) +cap(CAP_PAM, PII_NO, max_size=50*KB) cap(CAP_PERSISTENT_STATS, PII_MAYBE, max_size=50*MB, max_time=60) cap(CAP_PROCESS_LIST, PII_YES, max_size=30*KB, @@ -369,7 +374,7 @@ cap(CAP_X11_AUTH, PII_NO, max_size=100*KB) cap(CAP_XAPI_DEBUG, PII_MAYBE, max_size=10*MB) cap(CAP_XAPI_SUBPROCESS, PII_NO, max_size=5*KB, max_time=10) -cap(CAP_XENRT, PII_NO, min_size=0, max_size=5*KB, +cap(CAP_XENRT, PII_NO, min_size=0, max_size=500*MB, checked=False, hidden=True) cap(CAP_XENSERVER_CONFIG, PII_MAYBE, max_size=80*KB, max_time=5) @@ -615,7 +620,7 @@ exclude those logs from the archive. cmd_output(CAP_DISK_INFO, [PVS]) cmd_output(CAP_DISK_INFO, [VGS]) cmd_output(CAP_DISK_INFO, [LVS]) - file_output(CAP_DISK_INFO, [LVM_CACHE]) + file_output(CAP_DISK_INFO, [LVM_CACHE, LVM_CONFIG]) cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/scsi_host']) cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/scsi_disk']) cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/fc_transport']) @@ -656,9 +661,12 @@ exclude those logs from the archive. cmd_output(CAP_LOSETUP_A, [LOSETUP, '-a']) - file_output(CAP_MULTIPATH, [MULTIPATH_CONF]) - cmd_output(CAP_MULTIPATH, [DMSETUP, 'status']) + file_output(CAP_MULTIPATH, [MULTIPATH_CONF, MPP_CONF]) + cmd_output(CAP_MULTIPATH, [DMSETUP, 'table']) func_output(CAP_MULTIPATH, 'multipathd_topology', multipathd_topology) + cmd_output(CAP_MULTIPATH, [MPPUTIL, '-a']) + if CAP_MULTIPATH in entries: + dump_rdac_groups(CAP_MULTIPATH) file_output(CAP_NETWORK_CONFIG, [NETWORK_CONF]) file_output(CAP_NETWORK_CONFIG, [NETWORK_DBCACHE]) @@ -751,6 +759,7 @@ exclude those logs from the archive. tree_output(CAP_VNCTERM, VNCTERM_CORE_DIR) + cmd_output(CAP_WLB, [XE, 'pool-retrieve-wlb-configuration']) cmd_output(CAP_WLB, [XE, 'pool-retrieve-wlb-diagnostics']) tree_output(CAP_X11_LOGS, X11_LOGS_DIR, X11_LOGS_RE) @@ -761,6 +770,9 @@ exclude those logs from the archive. func_output(CAP_XAPI_SUBPROCESS, 'xapi_subprocesses', dump_xapi_subprocess_info) tree_output(CAP_XENRT, '/tmp', FIST_RE) + # CA-45540: capture QEMU core files + tree_output(CAP_XENRT, '/var/xen/qemu') + tree_output(CAP_XENRT, '/tmp', re.compile(r'^.*xen\.qemu-dm\.')) file_output(CAP_XENSERVER_CONFIG, [INITIAL_INVENTORY]) file_output(CAP_XENSERVER_CONFIG, [POOL_CONF, PTOKEN, XAPI_CONF, XAPI_SSL_CONF, @@ -772,7 +784,7 @@ exclude those logs from the archive. cmd_output(CAP_XENSERVER_CONFIG, [LS, '-lR', STATIC_VDIS]) func_output(CAP_XENSERVER_DATABASES, 'xapi-db.xml', dump_filtered_xapi_db) - cmd_output(CAP_XENSERVER_DATABASES, [XENSTORE_LS]) + cmd_output(CAP_XENSERVER_DATABASES, [XENSTORE_LS, '-f']) file_output(CAP_XENSERVER_DATABASES, [DB_CONF, DB_CONF_RIO, DB_DEFAULT_FIELDS, DB_SCHEMA_SQL]) tree_output(CAP_XENSERVER_DATABASES, OEM_CONFIG_DIR, OEM_DB_FILES_RE) file_output(CAP_XENSERVER_DATABASES, [XENSTORED_DB, XENSTORED_DB + '.bak']) @@ -973,18 +985,53 @@ def dump_xapi_rrds(cap): session.xenapi.session.logout() return output -def filter_db_pii(str, state): - if 'in_secret_table' not in state: - state['in_secret_table'] = False +'''Filter a Xapi XML database. + + There is one important assumption made in this class: + - the XML document does not contain any characters between the end of one + tag and the beginning of the next, ie every > is immediately followed by + a < +''' +class DBFilter: + def __init__(self): + self.result = '' + self.rest = '' + self.state = {} + + def filter_secrets(self, s): + if 'in_secret_table' not in self.state: + self.state['in_secret_table'] = False + + # this logic doesn't deal with properly!!! + if s.startswith('
'): + self.state['in_secret_table'] = False + + if self.state['in_secret_table'] and s.startswith("'): - state['in_secret_table'] = False - - if state['in_secret_table'] and str.startswith("') + while p != -1: + s = rem[:p+1] + rem = rem[p+1:] + self.result += self.filter_secrets(s) + p = rem.find('>') + self.rest = rem + + def output(self): + r = self.result + self.filter_secrets(self.rest) + self.result, self.rest = '', '' + self.state = {} + return r + +def filter_db_pii(s, state): + dbfilter = DBFilter() + dbfilter.feed(s) + return dbfilter.output() def dump_filtered_xapi_db(cap): db_file = None @@ -1018,18 +1065,12 @@ def dump_filtered_xapi_db(cap): if not ih: return '' - remain = '' + dbfilter = DBFilter() rec = ih.read(2048) while rec != '': - remain += rec - p = remain.find('>') - while p != -1: - str = remain[:p+1] - remain = remain[p+1:] - output += filter_db_pii(str, state) - p = remain.find('>') + dbfilter.feed(rec) rec = ih.read(2048) - output += remain + output = dbfilter.output() if pipe: pipe.wait() @@ -1088,24 +1129,12 @@ def csl_logs(cap): output = StringIO.StringIO() procs = [] - - def rotate_string(x, n): - transtbl = "" - for a in range(0, 256): - transtbl = transtbl + chr(a) - transtbl = transtbl[n:] + transtbl[0:n] - return x.translate(transtbl) - - def _untransform_string(str, remove_trailing_nulls=False): - """De-obfuscate string. To cope with an obfuscation bug in Rio, the argument - remove_trailing_nulls should be set to True""" - tmp = base64.decodestring(str) - if remove_trailing_nulls: - tmp = tmp.rstrip('\x00') - return rotate_string(tmp, -13) + csl_targets_fetched = [] for pbd in session.xenapi.PBD.get_all_records().values(): if pbd.has_key('device_config') and pbd['device_config'].has_key('target'): + if pbd['device_config']['target'] in csl_targets_fetched: + continue sr = session.xenapi.SR.get_record(pbd['SR']) if sr.has_key('type') and sr['type'] == 'cslg': if sr['shared'] and pbd['host'] != this_host and not i_am_master: @@ -1117,9 +1146,11 @@ def csl_logs(cap): server += ':' + dev_cfg['port'] if dev_cfg.has_key('username'): server += ',' + dev_cfg['username'] - if dev_cfg.has_key('password_transformed'): - server += ',' + _untransform_string(dev_cfg['password_transformed']) + if dev_cfg.has_key('password_secret'): + sec_ref = session.xenapi.secret.get_by_uuid(dev_cfg['password_secret']) + server += ',' + session.xenapi.secret.get_value(sec_ref) procs.append(ProcOutput([CSL, server, 'srv-log-get'], caps[cap][MAX_TIME], output)) + csl_targets_fetched.append(dev_cfg['target']) session.xenapi.session.logout() @@ -1175,6 +1206,23 @@ def fd_usage(cap): output += "%s: %s\n" % (k, str(fd_dict[k])) return output +def dump_rdac_groups(cap): + output = StringIO.StringIO() + procs = [ProcOutput([MPPUTIL, '-a'], caps[cap][MAX_TIME], output)] + + run_procs([procs]) + + if not procs[0].timed_out: + proc_line = 0 + for line in output.getvalue().splitlines(): + if line.startswith('ID'): + proc_line = 2 + elif line.startswith('----'): + proc_line -= 1 + elif proc_line > 0: + group, _ = line.split(None, 1) + cmd_output(cap, [MPPUTIL, '-g', group]) + def load_plugins(just_capabilities = False): def getText(nodelist): rc = "" @@ -1294,7 +1342,7 @@ def make_zip(subdir): if v.has_key('output'): zf.writestr(dest, v['output'].getvalue()) - else: + elif v.has_key('filename'): if os.stat(v['filename']).st_size < 50: compress_type = zipfile.ZIP_STORED else: -- 2.30.2