From a41754333f6faad72e0f7b12521de7b7f737f9a6 Mon Sep 17 00:00:00 2001 From: Gurucharan Shetty Date: Thu, 18 Oct 2012 15:44:37 -0700 Subject: [PATCH] ovs-ctl.in: Ability to save flows and kernel datapath config. Add a new command - "restart" to ovs-ctl. Calling this command will save and restore the Openflow flows on each bridge while stopping and starting the userspace daemons respectively. Also, during a force-reload-kmod, save the flows and kernel datapath configuration. Use the saved datapath configuration while readding the kernel module and the flows while starting the userspace daemons. Feature #13555. Signed-off-by: Gurucharan Shetty --- utilities/ovs-ctl.8 | 29 ++-- utilities/ovs-ctl.in | 101 +++++++++++-- utilities/ovs-save | 334 +++++++++++++++++++++++++++++-------------- 3 files changed, 336 insertions(+), 128 deletions(-) diff --git a/utilities/ovs-ctl.8 b/utilities/ovs-ctl.8 index 8cbd3cf8..3092d463 100644 --- a/utilities/ovs-ctl.8 +++ b/utilities/ovs-ctl.8 @@ -250,6 +250,14 @@ modules. This command does nothing and finishes successfully if the OVS daemons aren't running. . +.SH "The ``restart'' command" +. +.PP +The \fBrestart\fR command performs a \fBstop\fR followed by a \fBstart\fR +command. The command can take the same options as that of the \fBstart\fR +command. In addition, it saves and restores Openflow flows for each +individual bridge. +. .SH "The ``status'' command" . .PP @@ -279,27 +287,32 @@ implemented by Open vSwitch. The most common examples of these are bridge ``local ports''. . .IP 2. +Saves the Openflow flows of each bridge and the kernel datapath +configuration for each of the kernel datapaths. +. +.IP 3. Stops the Open vSwitch daemons, as if by a call to \fBovs\-ctl stop\fR. . -.IP 3. +.IP 4. Saves the kernel configuration state of the OVS internal interfaces listed in step 1, including IP and IPv6 addresses and routing table entries. . -.IP 4. +.IP 5. Unloads the Open vSwitch kernel module (including the bridge compatibility module if it is loaded). . -.IP 5. -Starts OVS back up, as if by a call to \fBovs\-ctl start\fR. This -reloads the kernel module and restarts the OVS daemons (including -\fBovs\-brcompatd\fR, if \fB\-\-brcompat\fR is specified). -. .IP 6. -Restores the kernel configuration state that was saved in step 3. +Starts OVS back up, as if by a call to \fBovs\-ctl start\fR. This +reloads the kernel module, restores the saved kernel datapath configuration, +restarts the OVS daemons (including \fBovs\-brcompatd\fR, if \fB\-\-brcompat\fR +is specified) and finally restores the saved Openflow flows. . .IP 7. +Restores the kernel configuration state that was saved in step 4. +. +.IP 8. Checks for daemons that may need to be restarted because they have packet sockets that are listening on old instances of Open vSwitch kernel interfaces and, if it finds any, prints a warning on stdout. diff --git a/utilities/ovs-ctl.in b/utilities/ovs-ctl.in index 2ead0042..9ce4973d 100755 --- a/utilities/ovs-ctl.in +++ b/utilities/ovs-ctl.in @@ -30,13 +30,21 @@ done ## start ## ## ----- ## +restore_datapaths () { + [ -n "${script_datapaths}" ] && \ + action "Restoring datapath configuration" "${script_datapaths}" +} + insert_openvswitch_mod_if_required () { # If openvswitch is already loaded then we're done. test -e /sys/module/openvswitch -o -e /sys/module/openvswitch_mod && \ return 0 # Load openvswitch. If that's successful then we're done. - action "Inserting openvswitch module" modprobe openvswitch && return 0 + if action "Inserting openvswitch module" modprobe openvswitch; then + restore_datapaths + return 0 + fi # If the bridge module is loaded, then that might be blocking # openvswitch. Try to unload it, if there are no bridges. @@ -50,6 +58,7 @@ insert_openvswitch_mod_if_required () { # Try loading openvswitch again. action "Inserting openvswitch module" modprobe openvswitch + restore_datapaths } insert_brcompat_mod_if_required () { @@ -293,14 +302,45 @@ internal_interfaces () { done } +save_flows () { + if set X `ovs_vsctl list-br`; then + shift + if "$datadir/scripts/ovs-save" save-flows "$@" > "$script_flows"; then + chmod +x "$script_flows" + return 0 + fi + fi + script_flows= + return 1 +} + save_interfaces () { - "$datadir/scripts/ovs-save" $ifaces > "$script" + "$datadir/scripts/ovs-save" save-interfaces ${ifaces} \ + > "${script_interfaces}" +} + +save_datapaths () { + "$datadir/scripts/ovs-save" save-datapaths ${datapaths} \ + > "${script_datapaths}" +} + +restore_flows () { + [ -n "${script_flows}" ] && \ + action "Restoring saved flows" "${script_flows}" } force_reload_kmod () { ifaces=`internal_interfaces` action "Detected internal interfaces: $ifaces" true + script_interfaces=`mktemp` + script_datapaths=`mktemp` + script_flows=`mktemp` + trap 'rm -f "${script_interfaces}" "${script_flows}" \ + "${script_datapaths}"' 0 1 2 13 15 + + action "Saving flows" save_flows + # Restart the database first, since a large database may take a # while to load, and we want to minimize forwarding disruption. stop_ovsdb @@ -308,8 +348,6 @@ force_reload_kmod () { stop_forwarding - script=`mktemp` - trap 'rm -f "$script"' 0 1 2 13 15 if action "Saving interface configuration" save_interfaces; then : else @@ -317,9 +355,18 @@ force_reload_kmod () { start_forwarding exit 1 fi - chmod +x "$script" + chmod +x "$script_interfaces" + + datapaths=`ovs-dpctl dump-dps` + if action "Saving datapath configuration" save_datapaths; then + chmod +x "${script_datapaths}" + else + log_warning_msg "Failed to save datapath configuration. The port\ + numbers may change after the restart" + script_datapaths="" + fi - for dp in `ovs-dpctl dump-dps`; do + for dp in ${datapaths}; do action "Removing datapath: $dp" ovs-dpctl del-dp "$dp" done @@ -337,7 +384,9 @@ force_reload_kmod () { start_forwarding - action "Restoring interface configuration" "$script" + restore_flows + + action "Restoring interface configuration" "$script_interfaces" rc=$? if test $rc = 0; then level=debug @@ -346,11 +395,32 @@ force_reload_kmod () { fi log="logger -p daemon.$level -t ovs-save" $log "force-reload-kmod interface restore script exited with status $rc:" - $log -f "$script" + $log -f "$script_interfaces" "$datadir/scripts/ovs-check-dead-ifs" } +## ------- ## +## restart ## +## ------- ## + +restart () { + script_flows=`mktemp` + trap 'rm -f "${script_flows}"' 0 1 2 13 15 + + action "Saving flows" save_flows + + # Restart the database first, since a large database may take a + # while to load, and we want to minimize forwarding disruption. + stop_ovsdb + start_ovsdb + + stop_forwarding + start_forwarding + + restore_flows +} + ## --------------- ## ## enable-protocol ## ## --------------- ## @@ -455,6 +525,7 @@ scripts. System administrators should not normally invoke it directly. Commands: start start Open vSwitch daemons stop stop Open vSwitch daemons + restart stop and start Open vSwitch daemons status check whether Open vSwitch daemons are running version print versions of Open vSwitch daemons load-kmod insert modules if not already present @@ -463,18 +534,18 @@ Commands: enable-protocol enable protocol specified in options with iptables help display this help message -One of the following options is required for "start" and "force-reload-kmod": +One of the following options is required for "start", "restart" and "force-reload-kmod": --system-id=UUID set specific ID to uniquely identify this system --system-id=random use a random but persistent UUID to identify this system -Other important options for "start" and "force-reload-kmod": +Other important options for "start", "restart" and "force-reload-kmod": --system-type=TYPE set system type (e.g. "XenServer") --system-version=VERSION set system version (e.g. "5.6.100-39265p") --external-id="key=value" add given key-value pair to Open_vSwitch external-ids --delete-bridges delete all bridges just before starting ovs-vswitchd -Less important options for "start" and "force-reload-kmod": +Less important options for "start", "restart" and "force-reload-kmod": --daemon-cwd=DIR set working dir for OVS daemons (default: $DAEMON_CWD) --no-force-corefiles do not force on core dumps for OVS daemons --no-mlockall do not lock all of ovs-vswitchd into memory @@ -482,13 +553,13 @@ Less important options for "start" and "force-reload-kmod": --ovs-vswitchd-priority=NICE set ovs-vswitchd's niceness (default: $OVS_VSWITCHD_PRIORITY) --ovs-brcompatd-priority=NICE set ovs-brcompatd's niceness (default: $OVS_BRCOMPATD_PRIORITY) -Debugging options for "start" and "force-reload-kmod": +Debugging options for "start", "restart" and "force-reload-kmod": --ovsdb-server-wrapper=WRAPPER --ovs-vswitchd-wrapper=WRAPPER --ovs-vswitchd-wrapper=WRAPPER run specified daemon under WRAPPER (either 'valgrind' or 'strace') -Options for "start", "force-reload-kmod", "load-kmod", "status", and "version": +Options for "start", "restart", "force-reload-kmod", "load-kmod", "status", and "version": --brcompat enable Linux bridge compatibility module and daemon File location options: @@ -606,6 +677,9 @@ case $command in stop_forwarding stop_ovsdb ;; + restart) + restart + ;; status) rc=0 for daemon in `daemons`; do @@ -639,4 +713,3 @@ case $command in exit 1 ;; esac - diff --git a/utilities/ovs-save b/utilities/ovs-save index 297c2fa3..9ed14ebb 100755 --- a/utilities/ovs-save +++ b/utilities/ovs-save @@ -14,20 +14,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -if test "X$1" = X--help; then +usage() { + UTIL=$(basename $0) cat <&2 - exit 1 -fi - -if test "$#" = 0; then - echo "# $0: no parameters given (use \"$0 --help\" for help)" -fi - -devs=$* -for dev in $devs; do - state=`ip link show dev $dev` || continue - - echo "# $dev" - # Link state (Ethernet addresses, up/down, ...) - linkcmd= - case $state in - *"state UP"* | *[,\<]"UP"[,\>]* ) - linkcmd="$linkcmd up" - ;; - *"state DOWN"*) - linkcmd="$linkcmd down" - ;; - esac - if expr "$state" : '.*\bdynamic\b' > /dev/null; then - linkcmd="$linkcmd dynamic" - fi - if qlen=`expr "$state" : '.*qlen \([0-9]+\)'`; then - linkcmd="$linkcmd txqueuelen $qlen" - fi - if hwaddr=`expr "$state" : '.*link/ether \([^ ]*\)'`; then - linkcmd="$linkcmd address $hwaddr" - fi - if brd=`expr "$state" : '.*brd \([^ ]*\)'`; then - linkcmd="$linkcmd broadcast $brd" - fi - if mtu=`expr "$state" : '.*mtu \([0-9]+\)'`; then - linkcmd="$linkcmd mtu $mtu" + +save_interfaces () { + if missing_program ip; then + echo "$0: ip not found in $PATH" >&2 + exit 1 fi - if test -n "$linkcmd"; then - echo ip link set dev $dev down # Required to change hwaddr. - echo ip link set dev $dev $linkcmd + + if test "$#" = 0; then + exit 0 fi - # IP addresses (including IPv6). - echo "ip addr flush dev $dev 2>/dev/null" # Suppresses "Nothing to flush". - ip addr show dev $dev | while read addr; do - set -- $addr - - # Check and trim family. - family=$1 - shift - case $family in - inet | inet6) ;; - *) continue ;; + devs="$@" + for dev in $devs; do + state=`ip link show dev $dev` || continue + + echo "# $dev" + # Link state (Ethernet addresses, up/down, ...) + linkcmd= + case $state in + *"state UP"* | *[,\<]"UP"[,\>]* ) + linkcmd="$linkcmd up" + ;; + *"state DOWN"*) + linkcmd="$linkcmd down" + ;; esac + if expr "$state" : '.*\bdynamic\b' > /dev/null; then + linkcmd="$linkcmd dynamic" + fi + if qlen=`expr "$state" : '.*qlen \([0-9]+\)'`; then + linkcmd="$linkcmd txqueuelen $qlen" + fi + if hwaddr=`expr "$state" : '.*link/ether \([^ ]*\)'`; then + linkcmd="$linkcmd address $hwaddr" + fi + if brd=`expr "$state" : '.*brd \([^ ]*\)'`; then + linkcmd="$linkcmd broadcast $brd" + fi + if mtu=`expr "$state" : '.*mtu \([0-9]+\)'`; then + linkcmd="$linkcmd mtu $mtu" + fi + if test -n "$linkcmd"; then + echo ip link set dev $dev down # Required to change hwaddr. + echo ip link set dev $dev $linkcmd + fi + + # IP addresses (including IPv6). + echo "ip addr flush dev $dev 2>/dev/null" # Suppresses "Nothing to flush". + ip addr show dev $dev | while read addr; do + set -- $addr + + # Check and trim family. + family=$1 + shift + case $family in + inet | inet6) ;; + *) continue ;; + esac - # Trim device off the end--"ip" insists on having "dev" precede it. - addrcmd= - while test $# != 0; do - case $1 in - dynamic) - # Omit kernel-maintained route. - continue 2 - ;; - scope) - if test "$2" = link; then - # Omit route derived from IP address, e.g. - # 172.16.0.0/16 derived from 172.16.12.34. + # Trim device off the end--"ip" insists on having "dev" precede it. + addrcmd= + while test $# != 0; do + case $1 in + dynamic) + # Omit kernel-maintained route. continue 2 - fi - ;; - "$dev"|"$dev:"*) - # Address label string - addrcmd="$addrcmd label $1" - shift - continue - ;; + ;; + scope) + if test "$2" = link; then + # Omit route derived from IP address, e.g. + # 172.16.0.0/16 derived from 172.16.12.34. + continue 2 + fi + ;; + "$dev"|"$dev:"*) + # Address label string + addrcmd="$addrcmd label $1" + shift + continue + ;; + esac + addrcmd="$addrcmd $1" + shift + done + if test "$1" != "$dev"; then + addrcmd="$addrcmd $1" + fi + + echo ip -f $family addr add $addrcmd dev $dev + done + + # Routes. + echo "ip route flush dev $dev proto boot 2>/dev/null" # Suppresses "Nothing to flush". + ip route show dev $dev | while read route; do + # "proto kernel" routes are installed by the kernel automatically. + case $route in + *" proto kernel "*) continue ;; esac - addrcmd="$addrcmd $1" - shift + + echo "ip route add $route dev $dev" done - if test "$1" != "$dev"; then - addrcmd="$addrcmd $1" - fi - echo ip -f $family addr add $addrcmd dev $dev + echo done - # Routes. - echo "ip route flush dev $dev proto boot 2>/dev/null" # Suppresses "Nothing to flush". - ip route show dev $dev | while read route; do - # "proto kernel" routes are installed by the kernel automatically. - case $route in - *" proto kernel "*) continue ;; - esac + if missing_program iptables-save; then + echo "# iptables-save not found in $PATH, not saving iptables state" + else + echo "# global" + echo "iptables-restore <<'EOF'" + iptables-save + echo "EOF" + fi +} + +save_flows () { + if missing_program ovs-ofctl; then + echo "$0: ovs-ofctl not found in $PATH" >&2 + exit 1 + fi - echo "ip route add $route dev $dev" + for bridge in "$@"; do + echo "ovs-ofctl add-flows ${bridge} - << EOF" + ovs-ofctl dump-flows "${bridge}" | sed -e '/NXST_FLOW/d' \ + -e 's/\(idle\|hard\)_age=[^,]*,//g' + echo "EOF" done +} - echo -done +save_datapaths () { + if missing_program ovs-dpctl; then + echo "$0: ovs-dpctl not found in $PATH" >&2 + exit 1 + fi + if missing_program ovs-vsctl; then + echo "$0: ovs-vsctl not found in $PATH" >&2 + exit 1 + fi + + for dp in "$@"; do + echo "ovs-dpctl add-dp ${dp}" + ovs-dpctl show $dp | while read line; do + # An example 'ovs-dpctl show' output looks like this: + # system@br1: + # lookups: hit:0 missed:0 lost:0 + # flows: 0 + # port 0: br1 (internal) + # port 2: gre2886795521 (ipsec_gre: key=flow, pmtud=false, remote_ip=172.17.1.1, tos=inherit) + # port 3: gre1 (ipsec_gre: remote_ip=192.168.113.1) + # port 14: gre2 (gre: remote_ip=192.168.115.1) + # port 15: gre3 (gre64: remote_ip=192.168.116.1) + # port 16: eth0 + # port 17: br1- (patch: peer=br1+) -if missing_program iptables-save; then - echo "# iptables-save not found in $PATH, not saving iptables state" -else - echo "# global" - echo "iptables-restore <<'EOF'" - iptables-save - echo "EOF" -fi + # Skip lines which do not have 'port' + if port_no=`expr "${line}" : '.*port \([0-9]\+\):'`; then :; else + continue + fi + + netdev=`echo ${line} | awk '{print $3}'` + + # Do not add port that has the same name as the datapath. It gets + # added by default. + [ "${dp#system@}" = "${netdev}" ] && continue + + type=`echo ${line} | awk '{print $4}' | sed 's/[:)(]//g'` + [ ! -n "${type}" ] && type="system" + + command="ovs-dpctl add-if ${dp}\ + ${netdev},type=${type},port_no=${port_no}" + + options=`echo ${line} | awk -F: '{print $3}' | sed 's/[) ]//g'` + [ -n "${options}" ] && command="${command},${options}" + + # For ipsec, ovs-dpctl does not show the key value pairs related + # to certificates. Get that information from ovs-vsctl. + if [ "${type}" = "ipsec_gre" ] ; then + if peer_cert=`ovs-vsctl get interface \ + "${netdev}" options:peer_cert 2>/dev/null`; then + # The option peer_cert comes with an accompanying + # "certificate" or "use_ssl_cert" + if certificate=`ovs-vsctl get interface "${netdev}" \ + options:certificate 2>/dev/null` ; then + command="${command},peer_cert=${peer_cert},certificate=${certificate}" + else + use_ssl_cert=`ovs-vsctl get interface "${netdev}" \ + options:use_ssl_cert 2>/dev/null` + command="${command},peer_cert=${peer_cert},use_ssl_cert=${use_ssl_cert}" + fi + else + psk=`ovs-vsctl get interface "${netdev}" \ + options:psk 2>/dev/null` + command="${command},psk=${psk}" + fi + fi + echo ${command} + done + done +} + +while [ $# -ne 0 ] +do + case $1 in + "save-datapaths") + shift + save_datapaths "$@" + exit 0 + ;; + "save-flows") + shift + save_flows "$@" + exit 0 + ;; + "save-interfaces") + shift + save_interfaces "$@" + exit 0 + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo >&2 "$0: unknown command \"$1\" (use --help for help)" + exit 1 + ;; + esac +done exit 0 -- 2.30.2