xenserver: Merge upstream changes and drop pre-5.6.100 support.
authorAndrew Evans <aevans@nicira.com>
Fri, 11 Feb 2011 00:50:49 +0000 (16:50 -0800)
committerAndrew Evans <aevans@nicira.com>
Fri, 11 Feb 2011 21:03:06 +0000 (13:03 -0800)
Citrix have made modifications to the various interface configuration scripts
that OVS replaces, so at present those modifications are lost when OVS is
installed on e.g. XenServer 5.6.100. This commit applies those changes while
preserving OVS-specific modifications that have been made in the interim.

One major change introduced by these updates is the removal of support for
XenServer releases prior to 5.6.100.

This commit also updates the OVS-supplied xen-bugtool in similar fashion. We
will soon remove xen-bugtool from OVS altogether and move the added OVS
functionality to an XML extension as described in the Supplemental Pack DDK
guide.

INSTALL.Linux
INSTALL.XenServer
tests/interface-reconfigure.at
vswitchd/vswitch.xml
xenserver/README
xenserver/etc_xensource_scripts_vif
xenserver/opt_xensource_libexec_InterfaceReconfigure.py
xenserver/opt_xensource_libexec_InterfaceReconfigureBridge.py
xenserver/opt_xensource_libexec_InterfaceReconfigureVswitch.py
xenserver/opt_xensource_libexec_interface-reconfigure
xenserver/usr_sbin_xen-bugtool

index 9d69bd864d7cd7138b1ca694e4e33883848b59cf..8ba73ed353ab5a7b816853ad19f19eda4ac6d783 100644 (file)
@@ -3,7 +3,7 @@
 
 This document describes how to build and install Open vSwitch on a
 generic Linux host.  If you want to install Open vSwitch on a Citrix
-XenServer version 5.5.0, see INSTALL.XenServer instead.
+XenServer, see INSTALL.XenServer instead.
 
 This version of Open vSwitch may be built manually with "configure"
 and "make", as described below.  You may also build Debian packages by
index 33b7f8ba52316bd71f1ed39deee571191174e3b5..e7be494cd69316564b2ca8623a7b7792bcaf8b87 100644 (file)
@@ -5,8 +5,7 @@ This document describes how to build and install Open vSwitch on a
 Citrix XenServer host.  If you want to install Open vSwitch on a
 generic Linux host, see INSTALL.Linux instead.
 
-These instructions have been tested with XenServer versions 5.5.0,
-5.5.0-24648p (Update 1), 5.5.0-25727p (Update 2), and 5.6.0.
+These instructions have been tested with XenServer 5.6 FP1.
 
 Building Open vSwitch for XenServer
 -----------------------------------
@@ -38,7 +37,7 @@ where:
     name of the Open vSwitch tarball, e.g. 0.90.0.
 
     <Xen kernel version> is the version number of the Xen kernel,
-    e.g. 2.6.18-128.1.1.el5.xs5.5.0.487.1006xen.  This version number
+    e.g. 2.6.32.12-0.7.1.xs5.6.100.307.170586xen.  This version number
     appears as the name of a directory in /lib/modules inside the VM.
     It always ends in "xen".
 
index bb7eff766b02dd3274799c595038499c5727c6f1..f47ca98f57e84bc223546e75fb4d3d16ef4bf557 100644 (file)
@@ -73,6 +73,7 @@ EOF
 <xenserver-network-configuration>
        <pif ref="OpaqueRef:e0955887-571f-17fc-a971-61c1ec7d81b6">
                <VLAN_slave_of/>
+                <tunnel_access_PIF_of/>
                <management>
                        False
                </management>
@@ -120,6 +121,7 @@ EOF
        </pif>
        <pif ref="OpaqueRef:eea8da94-a5e6-18fc-34a7-5e9b5a235806">
                <VLAN_slave_of/>
+                <tunnel_access_PIF_of/>
                <management>
                        False
                </management>
@@ -167,6 +169,7 @@ EOF
        </pif>
        <pif ref="OpaqueRef:2956e6c8-487e-981c-85ff-c84796418768">
                <VLAN_slave_of/>
+                <tunnel_access_PIF_of/>
                <management>
                        False
                </management>
@@ -214,6 +217,7 @@ EOF
        </pif>
        <pif ref="OpaqueRef:d2d1e51e-4da9-3163-8f57-bb683429335e">
                <VLAN_slave_of/>
+                <tunnel_access_PIF_of/>
                <management>
                        False
                </management>
@@ -265,6 +269,7 @@ EOF
                                OpaqueRef:e623e1d6-cd02-be8d-820d-49d65c710297
                        </master>
                </VLAN_slave_of>
+                <tunnel_access_PIF_of/>
                <management>
                        False
                </management>
@@ -312,6 +317,7 @@ EOF
        </pif>
        <pif ref="OpaqueRef:205d1186-2cd1-d5e6-45e4-ea1698ea6e15">
                <VLAN_slave_of/>
+                <tunnel_access_PIF_of/>
                <management>
                        True
                </management>
@@ -359,6 +365,7 @@ EOF
        </pif>
        <pif ref="OpaqueRef:8e3e37e6-ebb9-087e-0201-f6a56bf554c3">
                <VLAN_slave_of/>
+                <tunnel_access_PIF_of/>
                <management>
                        False
                </management>
@@ -410,6 +417,7 @@ EOF
                                OpaqueRef:4c0eb823-4d96-da1d-e75f-411b85badb0c
                        </master>
                </VLAN_slave_of>
+                <tunnel_access_PIF_of/>
                <management>
                        False
                </management>
@@ -656,6 +664,7 @@ Applying changes to /etc/sysconfig/network-scripts/route-xenbr2 configuration
 Applying changes to /etc/sysconfig/network configuration
 Applying changes to /etc/sysconfig/network-scripts/ifcfg-xenbr2 configuration
 /sbin/ifconfig eth2 up mtu 1500
+/sbin/ethtool -K eth2 gro off lro off
 /usr/bin/ovs-vsctl --timeout=20
     --with-iface --if-exists del-port eth2
     --may-exist add-br xenbr2
@@ -720,6 +729,7 @@ configure_datapath: extra bonds - []
 Applying changes to /etc/sysconfig/network-scripts/route-xapi3 configuration
 Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi3 configuration
 /sbin/ifconfig eth3 up mtu 1500
+/sbin/ethtool -K eth3 gro off lro off
 /usr/bin/ovs-vsctl --timeout=20
     --with-iface --if-exists del-port eth3
     --may-exist add-br xenbr3
@@ -800,7 +810,9 @@ netdev: down: device xenbr1 does not exist, ignoring
 Applying changes to /etc/sysconfig/network-scripts/route-xapi1 configuration
 Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi1 configuration
 /sbin/ifconfig eth0 up mtu 1500
+/sbin/ethtool -K eth0 gro off lro off
 /sbin/ifconfig eth1 up mtu 1500
+/sbin/ethtool -K eth1 gro off lro off
 /usr/bin/ovs-vsctl --timeout=20
     --if-exists del-br xenbr0
     --if-exists del-br xenbr1
@@ -882,7 +894,9 @@ netdev: down: device xenbr1 does not exist, ignoring
 Applying changes to /etc/sysconfig/network-scripts/route-xapi2 configuration
 Applying changes to /etc/sysconfig/network-scripts/ifcfg-xapi2 configuration
 /sbin/ifconfig eth0 up mtu 1500
+/sbin/ethtool -K eth0 gro off lro off
 /sbin/ifconfig eth1 up mtu 1500
+/sbin/ethtool -K eth1 gro off lro off
 /usr/bin/ovs-vsctl --timeout=20
     --if-exists del-br xenbr0
     --if-exists del-br xenbr1
index f4515f4fe175f3f5b4b7f80a01eebb561305768a..c80e8cdb5428fc840228f4b392c27d5c31d2a106 100644 (file)
       <column name="system_version">
         <p>
           The version of the system identified by <ref column="system_type"/>,
-          e.g. <code>5.5.0-24648p</code> on XenServer 5.5.0 build 24648.
+          e.g. <code>5.6.100-39265p</code> on XenServer 5.6.100 build 39265.
         </p>
         <p>
           System integrators are responsible for choosing and setting an
index 7da3ac2663e48b6a0d4f78fae2016ad4624384b0..3110aabc8d4ed145f1c52c3c7fb0a730261edd9f 100644 (file)
@@ -79,8 +79,8 @@ files are:
     uuid.py
 
         This is uuid.py from Python 2.5.  It is installed into the
-        Open vSwitch RPM because XenServer 5.5 and 5.6 use Python 2.4,
-        which do not have uuid.py.
+        Open vSwitch RPM because XenServer uses Python 2.4, which does
+        not have uuid.py.
 
 To install, build the Open vSwitch RPM with a command like this:
 
@@ -94,7 +94,7 @@ question and reboot them.  (The openvswitch-dbg package that is also
 produced need not be installed, but it is harmless to do so.)
 
 ----------------------------------------------------------------------
-Copyright (C) 2009, 2010 Nicira Networks, Inc.
+Copyright (C) 2009, 2010, 2011 Nicira Networks, Inc.
 
 Copying and distribution of this file, with or without modification,
 are permitted in any medium without royalty provided the copyright
index 88006e2c4901104c5874e21d435590e062b33f25..5675cdfe83b0dfc8d968d3221e4ebb54bd2c70a0 100755 (executable)
@@ -25,16 +25,6 @@ IP="/sbin/ip"
 
 vsctl="/usr/bin/ovs-vsctl"
 
-# XAPI before build 29381 (approximately) did not provide some of the
-# data in XenStore that we rely on.
-. /etc/xensource-inventory
-if test "$PRODUCT_VERSION" = "5.5.0" || test "${BUILD_NUMBER%[a-z]}" -le 26131
-then
-    xs550=true
-else
-    xs550=false
-fi
-
 handle_promiscuous()
 {
     local arg=$(xenstore-read "${PRIVATE}/other-config/promiscuous" 2>/dev/null)
@@ -96,43 +86,19 @@ handle_vswitch_vif_details()
     fi
 
     local vif_uuid=$(xenstore-read "${PRIVATE}/vif-uuid" 2>/dev/null)
-    if $xs550 && [ -z "${vif_uuid}" ] && [ -n "${vm_uuid}" ]; then
-        vif_uuid=$(xe vif-list --minimal vm-uuid="${vm_uuid}" device=$DEVID)
-    fi
     if [ -n "${vif_uuid}" ] ; then
-        set_vif_external_id "xs-vif-uuid" "${vif_uuid}"
+       set_vif_external_id "xs-vif-uuid" "${vif_uuid}"
     fi
 
     local vif_details=
     local net_uuid=$(xenstore-read "${PRIVATE}/network-uuid" 2>/dev/null)
-    if $xs550 && [ -z "${net_uuid}" ] && [ -n "${vif_uuid}" ]; then
-        net_uuid=$(xe vif-param-get uuid="${vif_uuid}" param-name=network-uuid)
-    fi
     if [ -n "${net_uuid}" ] ; then
-        set_vif_external_id "xs-network-uuid" "${net_uuid}"
+       set_vif_external_id "xs-network-uuid" "${net_uuid}"
     fi
-
     local address=$(xenstore-read "/local/domain/$DOMID/device/vif/$DEVID/mac" 2>/dev/null)
     if [ -n "${address}" ] ; then
         set_vif_external_id "attached-mac" "${address}"
     fi
-
-    if $xs550; then
-        # vNetManager needs to know the network UUID(s) associated with each
-        # datapath.  Normally interface-reconfigure adds them, but XAPI does
-        # not use interface-reconfigure for internal networks. Instead, XAPI
-        # calls the addbr ioctl internally, so we have to do it here instead
-        # for internal networks.  This is only acceptable because xapi is lazy
-        # about creating internal networks: it only creates one just before it
-        # adds the first vif to it.  There may still be a brief delay between
-        # the initial ovs-vswitchd connection to vNetManager and setting this
-        # configuration variable, but vNetManager can tolerate that.
-        local bridge=$1
-        if [ -n "${net_uuid}" ] ; then
-            logger -t scripts-vif "${bridge} xs-network-uuids ${net_uuid}"
-            echo "-- br-set-external-id $bridge xs-network-uuids ${net_uuid}"
-        fi
-    fi
 }
 
 add_to_bridge()
index 68f7204cb52f5b26c9ef039c624ba68750fe9d94..a9bbf07e2f80b3b7c4a6971048111672febe6b20 100644 (file)
@@ -279,7 +279,7 @@ _BOND_XML_TAG = "bond"
 _NETWORK_XML_TAG = "network"
 _POOL_XML_TAG = "pool"
 
-_ETHTOOL_OTHERCONFIG_ATTRS = ['ethtool-%s' % x for x in 'autoneg', 'speed', 'duplex', 'rx', 'tx', 'sg', 'tso', 'ufo', 'gso' ]
+_ETHTOOL_OTHERCONFIG_ATTRS = ['ethtool-%s' % x for x in 'autoneg', 'speed', 'duplex', 'rx', 'tx', 'sg', 'tso', 'ufo', 'gso', 'gro', 'lro' ]
 
 _PIF_OTHERCONFIG_ATTRS = [ 'domain', 'peerdns', 'defaultroute', 'mtu', 'static-routes' ] + \
                         [ 'bond-%s' % x for x in 'mode', 'miimon', 'downdelay',
@@ -391,12 +391,7 @@ class DatabaseCache(object):
                 continue
             self.__pifs[p] = {}
             for f in _PIF_ATTRS:
-                if f in [ "tunnel_access_PIF_of", "tunnel_transport_PIF_of" ] and f not in rec:
-                    # XenServer 5.5 network records did not have
-                    # these fields, so allow them to be missing.
-                    pass
-                else:
-                    self.__pifs[p][f] = rec[f]
+                self.__pifs[p][f] = rec[f]
             self.__pifs[p]['other_config'] = {}
             for f in _PIF_OTHERCONFIG_ATTRS:
                 if not rec['other_config'].has_key(f): continue
@@ -404,8 +399,7 @@ class DatabaseCache(object):
 
     def __get_vlan_records_from_xapi(self, session):
         self.__vlans = {}
-        for v in session.xenapi.VLAN.get_all():
-            rec = session.xenapi.VLAN.get_record(v)
+        for (v,rec) in session.xenapi.VLAN.get_all_records().items():
             if not self.__pif_on_host(rec['untagged_PIF']):
                 continue
             self.__vlans[v] = {}
@@ -424,8 +418,7 @@ class DatabaseCache(object):
 
     def __get_bond_records_from_xapi(self, session):
         self.__bonds = {}
-        for b in session.xenapi.Bond.get_all():
-            rec = session.xenapi.Bond.get_record(b)
+        for (b,rec) in session.xenapi.Bond.get_all_records().items():
             if not self.__pif_on_host(rec['master']):
                 continue
             self.__bonds[b] = {}
@@ -434,8 +427,7 @@ class DatabaseCache(object):
 
     def __get_network_records_from_xapi(self, session):
         self.__networks = {}
-        for n in session.xenapi.network.get_all():
-            rec = session.xenapi.network.get_record(n)
+        for (n,rec) in session.xenapi.network.get_all_records().items():
             self.__networks[n] = {}
             for f in _NETWORK_ATTRS:
                 if f == "PIFs":
@@ -512,13 +504,7 @@ class DatabaseCache(object):
 
                 self.__get_pif_records_from_xapi(session, host)
 
-                try:
-                    self.__get_tunnel_records_from_xapi(session)
-                except XenAPI.Failure, e:
-                    error,details = e.details
-                    if error == "MESSAGE_METHOD_UNKNOWN" and details == "tunnel.get_all":
-                        pass
-
+                self.__get_tunnel_records_from_xapi(session)
                 self.__get_pool_records_from_xapi(session)
                 self.__get_vlan_records_from_xapi(session)
                 self.__get_bond_records_from_xapi(session)
@@ -677,8 +663,9 @@ class DatabaseCache(object):
 #
 #
 #
+PIF_OTHERCONFIG_DEFAULTS = {'gro': 'off', 'lro': 'off'}
 
-def ethtool_settings(oc):
+def ethtool_settings(oc, defaults = {}):
     settings = []
     if oc.has_key('ethtool-speed'):
         val = oc['ethtool-speed']
@@ -688,8 +675,8 @@ def ethtool_settings(oc):
             log("Invalid value for ethtool-speed = %s. Must be 10|100|1000." % val)
     if oc.has_key('ethtool-duplex'):
         val = oc['ethtool-duplex']
-        if val in ["10", "100", "1000"]:
-            settings += ['duplex', 'val']
+        if val in ["half", "full"]:
+            settings += ['duplex', val]
         else:
             log("Invalid value for ethtool-duplex = %s. Must be half|full." % val)
     if oc.has_key('ethtool-autoneg'):
@@ -701,7 +688,7 @@ def ethtool_settings(oc):
         else:
             log("Invalid value for ethtool-autoneg = %s. Must be on|true|off|false." % val)
     offload = []
-    for opt in ("rx", "tx", "sg", "tso", "ufo", "gso"):
+    for opt in ("rx", "tx", "sg", "tso", "ufo", "gso", "gro", "lro"):
         if oc.has_key("ethtool-" + opt):
             val = oc["ethtool-" + opt]
             if val in ["true", "on"]:
@@ -710,6 +697,8 @@ def ethtool_settings(oc):
                 offload += [opt, 'off']
             else:
                 log("Invalid value for ethtool-%s = %s. Must be on|true|off|false." % (opt, val))
+        elif opt in defaults:
+            offload += [opt, defaults[opt]]
     return settings,offload
 
 # By default the MTU is taken from the Network.MTU setting for VIF,
@@ -882,8 +871,7 @@ def pif_get_vlan_masters(pif):
 # Tunnel PIFs
 #
 def pif_is_tunnel(pif):
-    rec = db().get_pif_record(pif)
-    return rec.has_key('tunnel_access_PIF_of') and len(rec['tunnel_access_PIF_of']) > 0
+    return len(db().get_pif_record(pif)['tunnel_access_PIF_of']) > 0
 
 #
 # Datapath base class
index 8803122d8a7ec86dfe46764ac9b14e14c7f74032..a93e438667debc347d9248b4969b305db6e25a21 100644 (file)
@@ -274,7 +274,8 @@ def _configure_physical_interface(pif):
     f.write("TYPE=Ethernet\n")
     f.write("HWADDR=%(MAC)s\n" % pifrec)
 
-    settings,offload = ethtool_settings(pifrec['other_config'])
+    settings,offload = ethtool_settings(pifrec['other_config'],
+                                        PIF_OTHERCONFIG_DEFAULTS)
     if len(settings):
         f.write("ETHTOOL_OPTS=\"%s\"\n" % str.join(" ", settings))
     if len(offload):
@@ -338,7 +339,7 @@ def _configure_bond_interface(pif):
     if len(offload):
         f.write("ETHTOOL_OFFLOAD_OPTS=\"%s\"\n" % str.join(" ", offload))
 
-    mtu = mtu_setting(pifrec['network'], "VLAN-PIF", pifrec['other_config'])
+    mtu = mtu_setting(pifrec['network'], "Bond-PIF", pifrec['other_config'])
     if mtu:
         f.write("MTU=%s\n" % mtu)
 
@@ -389,7 +390,7 @@ def _configure_vlan_interface(pif):
     if len(offload):
         f.write("ETHTOOL_OFFLOAD_OPTS=\"%s\"\n" % str.join(" ", offload))
 
-    mtu = mtu_setting(pifrec['network'], "Bond-PIF", pifrec['other_config'])
+    mtu = mtu_setting(pifrec['network'], "VLAN-PIF", pifrec['other_config'])
     if mtu:
         f.write("MTU=%s\n" % mtu)
 
index 33beb76c2d922d25b8a3d7ed59081386b5aaae85..928e6cefa6f22cfede67d61b78dc6917080a9bea 100644 (file)
@@ -529,7 +529,7 @@ class DatapathVswitch(Datapath):
 
             netdev_up(dev, mtu)
 
-            settings, offload = ethtool_settings(oc)
+            settings, offload = ethtool_settings(oc, PIF_OTHERCONFIG_DEFAULTS)
             if len(settings):
                 run_command(['/sbin/ethtool', '-s', dev] + settings)
             if len(offload):
index f1c1336c8e9293bd97fa57cb5ed280bc3737f393..ff4b97669c12caa7fe2f2e361546c518c33037bb 100755 (executable)
@@ -527,6 +527,8 @@ def action_force_rewrite(bridge, config):
     f.write('\t\t<VLAN_slave_of/>\n')
     f.write('\t\t<VLAN_master_of>OpaqueRef:NULL</VLAN_master_of>\n')
     f.write('\t\t<VLAN>-1</VLAN>\n')
+    f.write('\t\t<tunnel_access_PIF_of/>\n')
+    f.write('\t\t<tunnel_transport_PIF_of/>\n')
     f.write('\t\t<device>%s</device>\n' % interface)
     f.write('\t\t<MAC>%s</MAC>\n' % mac)
     f.write('\t\t<other_config/>\n')
index af88c74a5aa0db94ac3b482a05048cf054d97b4d..aeffed40d4c1d6cf9f680e3720f58c426dc96a29 100755 (executable)
@@ -79,7 +79,8 @@ FSTAB = '/etc/fstab'
 PROC_MOUNTS = '/proc/mounts'
 ISCSI_CONF = '/etc/iscsi/iscsid.conf'
 ISCSI_INITIATOR = '/etc/iscsi/initiatorname.iscsi'
-LVM_CACHE = '/etc/lvm/.cache'
+LVM_CACHE = '/etc/lvm/cache/.cache'
+LVM_CONFIG = '/etc/lvm/lvm.conf'
 PROC_CPUINFO = '/proc/cpuinfo'
 PROC_MEMINFO = '/proc/meminfo'
 PROC_IOPORTS = '/proc/ioports'
@@ -109,6 +110,7 @@ IFCFG_RE = re.compile(r'^.*/ifcfg-.*')
 ROUTE_RE = re.compile(r'^.*/route-.*')
 NETWORK_DBCACHE = '/var/xapi/network.dbcache'
 RESOLV_CONF = '/etc/resolv.conf'
+MPP_CONF = '/etc/mpp.conf'
 MULTIPATH_CONF = '/etc/multipath.conf'
 NSSWITCH_CONF = '/etc/nsswitch.conf'
 NTP_CONF = '/etc/ntp.conf'
@@ -152,11 +154,11 @@ INSTALLED_REPOS_DIR = '/etc/xensource/installed-repos'
 PATCH_APPLIED_DIR = '/var/patch/applied'
 XENSERVER_LOGS = \
     [ VAR_LOG_DIR + x for x in
-      ['xensource.log', 'audit.log', 'xenstored-access.log', 'SMlog', 'xen/xenstored-trace.log', 
+      ['xensource.log', 'audit.log', 'xenstored-access.log', 'SMlog', 'VMPRlog', 'xen/xenstored-trace.log', 
        'xen/xen-hotplug.log', 'xen/domain-builder-ng.log', 'squeezed.log',
        'openvswitch/ovs-brcompatd.log', 'openvswitch/ovs-vswitchd.log', 'openvswitch/ovsdb-server.log' ] +
       [ f % n for n in range(1, 20) \
-            for f in ['xensource.log.%d', 'xensource.log.%d.gz','SMlog.%d', 'SMlog.%d.gz',
+            for f in ['xensource.log.%d', 'xensource.log.%d.gz','SMlog.%d', 'SMlog.%d.gz', 'VMPRlog.%d', 'VMPRlog.%d.gz',
                       'audit.log.%d', 'audit.log.%d.gz', 'xenstored-access.log.%d', 'xenstored-access.log.%d.gz', \
                       'xen/xenstored-access.log.%d', 'xen/xenstored-access.log.%d.gz', 'squeezed.log.%d', \
                       'openvswitch/ovs-brcompatd.log.%d', 'openvswitch/ovs-brcompatd.log.%d.gz', \
@@ -211,6 +213,7 @@ LVS = '/usr/sbin/lvs'
 LVDISPLAY = '/usr/sbin/lvdisplay'
 MD5SUM = '/usr/bin/md5sum'
 MODINFO = '/sbin/modinfo'
+MPPUTIL = '/usr/sbin/mppUtil'
 MULTIPATHD = '/sbin/multipathd'
 NETSTAT = '/bin/netstat'
 OVS_DPCTL = '/usr/bin/ovs-dpctl'
@@ -322,6 +325,8 @@ dbg = False
 
 def cap(key, pii=PII_MAYBE, min_size=-1, max_size=-1, min_time=-1,
         max_time=-1, mime=MIME_TEXT, checked=True, hidden=False):
+    if  os.getenv('XEN_RT') and max_time > 0:
+        max_time *= 5
     caps[key] = (key, pii, min_size, max_size, min_time, max_time, mime,
                  checked, hidden)
     cap_sizes[key] = 0
@@ -331,11 +336,11 @@ cap(CAP_BLOBS,               PII_NO,                    max_size=5*MB)
 cap(CAP_BOOT_LOADER,         PII_NO,                    max_size=3*KB,
     max_time=5)
 cap(CAP_CVSM,                PII_NO,                    max_size=3*MB,
-    max_time=60)
-cap(CAP_DISK_INFO,           PII_MAYBE,                 max_size=25*KB,
+    max_time=120)
+cap(CAP_DISK_INFO,           PII_MAYBE,                 max_size=50*KB,
     max_time=20)
 cap(CAP_FIRSTBOOT,           PII_YES,   min_size=60*KB, max_size=80*KB)
-cap(CAP_HARDWARE_INFO,       PII_MAYBE,                 max_size=30*KB,
+cap(CAP_HARDWARE_INFO,       PII_MAYBE,                 max_size=50*KB,
     max_time=20)
 cap(CAP_HDPARM_T,            PII_NO,    min_size=0,     max_size=5*KB,
     min_time=20, max_time=90, checked=False, hidden=True)
@@ -345,13 +350,13 @@ cap(CAP_HOST_CRASHDUMP_LOGS, PII_NO)
 cap(CAP_KERNEL_INFO,         PII_MAYBE,                 max_size=120*KB,
     max_time=5)
 cap(CAP_LOSETUP_A,           PII_MAYBE,                 max_size=KB, max_time=5)
-cap(CAP_MULTIPATH,           PII_MAYBE,                 max_size=10*KB,
+cap(CAP_MULTIPATH,           PII_MAYBE,                 max_size=20*KB,
     max_time=10)
 cap(CAP_NETWORK_CONFIG,      PII_IF_CUSTOMIZED,
-                                        min_size=0,     max_size=20*KB)
+                                        min_size=0,     max_size=40*KB)
 cap(CAP_NETWORK_STATUS,      PII_YES,                   max_size=19*KB,
     max_time=30)
-cap(CAP_PAM,                 PII_NO,                    max_size=30*KB)
+cap(CAP_PAM,                 PII_NO,                    max_size=50*KB)
 cap(CAP_PERSISTENT_STATS,    PII_MAYBE,                 max_size=50*MB,
     max_time=60)
 cap(CAP_PROCESS_LIST,        PII_YES,                   max_size=30*KB,
@@ -369,7 +374,7 @@ cap(CAP_X11_AUTH,            PII_NO,                    max_size=100*KB)
 cap(CAP_XAPI_DEBUG,          PII_MAYBE,                 max_size=10*MB)
 cap(CAP_XAPI_SUBPROCESS,     PII_NO,                    max_size=5*KB,
     max_time=10)
-cap(CAP_XENRT,               PII_NO,    min_size=0,     max_size=5*KB,
+cap(CAP_XENRT,               PII_NO,    min_size=0,     max_size=500*MB,
     checked=False, hidden=True)
 cap(CAP_XENSERVER_CONFIG,    PII_MAYBE,                 max_size=80*KB,
     max_time=5)
@@ -615,7 +620,7 @@ exclude those logs from the archive.
     cmd_output(CAP_DISK_INFO, [PVS])
     cmd_output(CAP_DISK_INFO, [VGS])
     cmd_output(CAP_DISK_INFO, [LVS])
-    file_output(CAP_DISK_INFO, [LVM_CACHE])
+    file_output(CAP_DISK_INFO, [LVM_CACHE, LVM_CONFIG])
     cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/scsi_host'])
     cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/scsi_disk'])
     cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/fc_transport'])
@@ -656,9 +661,12 @@ exclude those logs from the archive.
 
     cmd_output(CAP_LOSETUP_A, [LOSETUP, '-a'])
 
-    file_output(CAP_MULTIPATH, [MULTIPATH_CONF])
-    cmd_output(CAP_MULTIPATH, [DMSETUP, 'status'])
+    file_output(CAP_MULTIPATH, [MULTIPATH_CONF, MPP_CONF])
+    cmd_output(CAP_MULTIPATH, [DMSETUP, 'table'])
     func_output(CAP_MULTIPATH, 'multipathd_topology', multipathd_topology)
+    cmd_output(CAP_MULTIPATH, [MPPUTIL, '-a'])
+    if CAP_MULTIPATH in entries:
+        dump_rdac_groups(CAP_MULTIPATH)
 
     file_output(CAP_NETWORK_CONFIG, [NETWORK_CONF])
     file_output(CAP_NETWORK_CONFIG, [NETWORK_DBCACHE])
@@ -751,6 +759,7 @@ exclude those logs from the archive.
 
     tree_output(CAP_VNCTERM, VNCTERM_CORE_DIR)
 
+    cmd_output(CAP_WLB, [XE, 'pool-retrieve-wlb-configuration'])
     cmd_output(CAP_WLB, [XE, 'pool-retrieve-wlb-diagnostics'])
 
     tree_output(CAP_X11_LOGS, X11_LOGS_DIR, X11_LOGS_RE)
@@ -761,6 +770,9 @@ exclude those logs from the archive.
     func_output(CAP_XAPI_SUBPROCESS, 'xapi_subprocesses', dump_xapi_subprocess_info)
 
     tree_output(CAP_XENRT, '/tmp', FIST_RE)
+    # CA-45540: capture QEMU core files
+    tree_output(CAP_XENRT, '/var/xen/qemu')
+    tree_output(CAP_XENRT, '/tmp', re.compile(r'^.*xen\.qemu-dm\.'))
 
     file_output(CAP_XENSERVER_CONFIG, [INITIAL_INVENTORY])
     file_output(CAP_XENSERVER_CONFIG, [POOL_CONF, PTOKEN, XAPI_CONF, XAPI_SSL_CONF, 
@@ -772,7 +784,7 @@ exclude those logs from the archive.
     cmd_output(CAP_XENSERVER_CONFIG, [LS, '-lR', STATIC_VDIS])
 
     func_output(CAP_XENSERVER_DATABASES, 'xapi-db.xml', dump_filtered_xapi_db)
-    cmd_output(CAP_XENSERVER_DATABASES, [XENSTORE_LS])
+    cmd_output(CAP_XENSERVER_DATABASES, [XENSTORE_LS, '-f'])
     file_output(CAP_XENSERVER_DATABASES, [DB_CONF, DB_CONF_RIO, DB_DEFAULT_FIELDS, DB_SCHEMA_SQL])
     tree_output(CAP_XENSERVER_DATABASES, OEM_CONFIG_DIR, OEM_DB_FILES_RE)
     file_output(CAP_XENSERVER_DATABASES, [XENSTORED_DB, XENSTORED_DB + '.bak'])
@@ -973,18 +985,53 @@ def dump_xapi_rrds(cap):
     session.xenapi.session.logout()
     return output
 
-def filter_db_pii(str, state):
-    if 'in_secret_table' not in state:
-        state['in_secret_table'] = False
+'''Filter a Xapi XML database.
+
+    There is one important assumption made in this class:
+    - the XML document does not contain any characters between the end of one
+      tag and the beginning of the next, ie every > is immediately followed by
+      a <
+'''
+class DBFilter:
+    def __init__(self):
+        self.result = ''
+        self.rest = ''
+        self.state = {}
+
+    def filter_secrets(self, s):
+        if 'in_secret_table' not in self.state:
+            self.state['in_secret_table'] = False
+
+        # this logic doesn't deal with <table name="secret" /> properly!!!
+        if s.startswith('<table ') and 'name="secret"' in s:
+            self.state['in_secret_table'] = True
+        elif s.startswith('</table>'):
+            self.state['in_secret_table'] = False
+        
+        if self.state['in_secret_table'] and s.startswith("<row"): # match only on DB rows
+            s = re.sub(r'(value=")[^"]+(")', r'\1REMOVED\2', s)
+        return s
 
-    if str.startswith('<table ') and 'name="secret"' in str:
-        state['in_secret_table'] = True
-    elif str.startswith('</table>'):
-        state['in_secret_table'] = False
-    
-    if state['in_secret_table'] and str.startswith("<row"): # match only on DB rows
-        str = re.sub(r'(value=")[^"]+(")', r'\1REMOVED\2', str)
-    return str
+    def feed(self, s):
+        rem = self.rest + s
+        p = rem.find('>')
+        while p != -1:
+            s = rem[:p+1]
+            rem = rem[p+1:]
+            self.result  += self.filter_secrets(s)
+            p = rem.find('>')
+        self.rest = rem
+
+    def output(self):
+        r = self.result + self.filter_secrets(self.rest)
+        self.result, self.rest = '', ''
+        self.state = {}
+        return r
+
+def filter_db_pii(s, state):
+    dbfilter = DBFilter()
+    dbfilter.feed(s)
+    return dbfilter.output()
 
 def dump_filtered_xapi_db(cap):
     db_file = None
@@ -1018,18 +1065,12 @@ def dump_filtered_xapi_db(cap):
     if not ih:
         return ''
 
-    remain = ''
+    dbfilter = DBFilter()
     rec = ih.read(2048)
     while rec != '':
-        remain += rec
-        p = remain.find('>')
-        while p != -1:
-            str = remain[:p+1]
-            remain = remain[p+1:]
-            output += filter_db_pii(str, state)
-            p = remain.find('>')
+        dbfilter.feed(rec)
         rec = ih.read(2048)
-    output += remain
+    output = dbfilter.output()
 
     if pipe:
         pipe.wait()
@@ -1088,24 +1129,12 @@ def csl_logs(cap):
 
     output = StringIO.StringIO()
     procs = []
-
-    def rotate_string(x, n):
-        transtbl = ""
-        for a in range(0, 256):
-            transtbl = transtbl + chr(a)
-        transtbl = transtbl[n:] + transtbl[0:n]
-        return x.translate(transtbl)
-
-    def _untransform_string(str, remove_trailing_nulls=False):
-        """De-obfuscate string. To cope with an obfuscation bug in Rio, the argument
-        remove_trailing_nulls should be set to True"""
-        tmp = base64.decodestring(str)
-        if remove_trailing_nulls:
-            tmp = tmp.rstrip('\x00')
-        return rotate_string(tmp, -13)
+    csl_targets_fetched = []
 
     for pbd in session.xenapi.PBD.get_all_records().values():
         if pbd.has_key('device_config') and pbd['device_config'].has_key('target'):
+            if pbd['device_config']['target'] in csl_targets_fetched:
+                continue
             sr = session.xenapi.SR.get_record(pbd['SR'])
             if sr.has_key('type') and sr['type'] == 'cslg':
                 if sr['shared'] and pbd['host'] != this_host and not i_am_master:
@@ -1117,9 +1146,11 @@ def csl_logs(cap):
                     server += ':' + dev_cfg['port']
                 if dev_cfg.has_key('username'):
                     server += ',' + dev_cfg['username']
-                if dev_cfg.has_key('password_transformed'):
-                    server += ',' + _untransform_string(dev_cfg['password_transformed'])
+                if dev_cfg.has_key('password_secret'):
+                    sec_ref = session.xenapi.secret.get_by_uuid(dev_cfg['password_secret'])
+                    server += ',' + session.xenapi.secret.get_value(sec_ref)
                 procs.append(ProcOutput([CSL, server, 'srv-log-get'], caps[cap][MAX_TIME], output))
+                csl_targets_fetched.append(dev_cfg['target'])
 
     session.xenapi.session.logout()
 
@@ -1175,6 +1206,23 @@ def fd_usage(cap):
         output += "%s: %s\n" % (k, str(fd_dict[k]))
     return output
 
+def dump_rdac_groups(cap):
+    output = StringIO.StringIO()
+    procs = [ProcOutput([MPPUTIL, '-a'], caps[cap][MAX_TIME], output)]
+
+    run_procs([procs])
+
+    if not procs[0].timed_out:
+        proc_line = 0
+        for line in output.getvalue().splitlines():
+            if line.startswith('ID'):
+                proc_line = 2
+            elif line.startswith('----'):
+                proc_line -= 1
+            elif proc_line > 0:
+                group, _ = line.split(None, 1)
+                cmd_output(cap, [MPPUTIL, '-g', group])
+
 def load_plugins(just_capabilities = False):
     def getText(nodelist):
         rc = ""
@@ -1294,7 +1342,7 @@ def make_zip(subdir):
             
                 if v.has_key('output'):
                     zf.writestr(dest, v['output'].getvalue())
-                else:
+                elif v.has_key('filename'):
                     if os.stat(v['filename']).st_size < 50:
                         compress_type = zipfile.ZIP_STORED
                     else: