From: Ben Pfaff Date: Tue, 30 Nov 2010 21:44:01 +0000 (-0800) Subject: Implement stress option framework. X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cc01d0bb79103a01aa0afc3484249fd518b877ea;p=openvswitch Implement stress option framework. Stress options allow developers testing Open vSwitch to trigger behavior that otherwise would occur only in corner cases. Developers and testers can thereby more easily discover bugs that would otherwise manifest only rarely or nondeterministically. Stress options may cause surprising behavior even when they do not actually reveal bugs, so they should only be enabled as part of testing Open vSwitch. This commit implements the framework and adds a few example stress options. This commit started from code written by Andrew Lambeth. Suggested-by: Henrik Amren CC: Andrew Lambeth --- diff --git a/AUTHORS b/AUTHORS index d283def2..bb675225 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,6 +1,7 @@ The following people, in alphabetical order, have either authored or signed off on commits in the Open vSwitch version control repository. +Andrew Lambeth wal@nicira.com Andy Southgate andy.southgate@citrix.com Ben Pfaff blp@nicira.com Bryan Phillippe bp@toroki.com diff --git a/lib/automake.mk b/lib/automake.mk index 91d8fff2..6d84a921 100644 --- a/lib/automake.mk +++ b/lib/automake.mk @@ -133,6 +133,8 @@ lib_libopenvswitch_a_SOURCES = \ lib/stream-unix.c \ lib/stream.c \ lib/stream.h \ + lib/stress.c \ + lib/stress.h \ lib/string.h \ lib/svec.c \ lib/svec.h \ @@ -220,6 +222,7 @@ EXTRA_DIST += \ lib/ssl-peer-ca-cert.man \ lib/ssl.man \ lib/ssl-syn.man \ + lib/stress-unixctl.man \ lib/unixctl.man \ lib/unixctl-syn.man \ lib/vconn-active.man \ @@ -261,6 +264,11 @@ lib/coverage.def: $(DIST_SOURCES) sed -n 's|^COVERAGE_DEFINE(\([_a-zA-Z0-9]\{1,\}\)).*$$|COVERAGE_COUNTER(\1)|p' $(all_sources) | LC_ALL=C sort -u > $@ CLEANFILES += lib/coverage.def +lib/stress.$(OBJEXT): lib/stress.def +lib/stress.def: $(DIST_SOURCES) + sed -n '/^STRESS_OPTION(/,/);$$/{s/);$$/)/;p}' $(all_sources) > $@ +CLEANFILES += lib/stress.def + lib/vlog.$(OBJEXT): lib/vlog-modules.def lib/vlog-modules.def: $(DIST_SOURCES) sed -n 's|^VLOG_DEFINE_\(THIS_\)\{0,1\}MODULE(\([_a-zA-Z0-9]\{1,\}\)).*$$|VLOG_MODULE(\2)|p' $(all_sources) | LC_ALL=C sort -u > $@ diff --git a/lib/netlink.c b/lib/netlink.c index 8806f91d..ba32ca3a 100644 --- a/lib/netlink.c +++ b/lib/netlink.c @@ -29,6 +29,7 @@ #include "netlink-protocol.h" #include "ofpbuf.h" #include "poll-loop.h" +#include "stress.h" #include "timeval.h" #include "util.h" #include "vlog.h" @@ -259,6 +260,15 @@ nl_sock_sendv(struct nl_sock *sock, const struct iovec iov[], size_t n_iov, return error; } +/* This stress option is useful for testing that OVS properly tolerates + * -ENOBUFS on NetLink sockets. Such errors are unavoidable because they can + * occur if the kernel cannot temporarily allocate enough GFP_ATOMIC memory to + * reply to a request. They can also occur if messages arrive on a multicast + * channel faster than OVS can process them. */ +STRESS_OPTION( + netlink_overflow, "simulate netlink socket receive buffer overflow", + 5, 1, -1, 100); + /* Tries to receive a netlink message from the kernel on 'sock'. If * successful, stores the received message into '*bufp' and returns 0. The * caller is responsible for destroying the message with ofpbuf_delete(). On @@ -340,9 +350,16 @@ try_again: ofpbuf_delete(buf); return EPROTO; } + + if (STRESS(netlink_overflow)) { + ofpbuf_delete(buf); + return ENOBUFS; + } + *bufp = buf; log_nlmsg(__func__, 0, buf->data, buf->size); COVERAGE_INC(netlink_received); + return 0; } diff --git a/lib/stream-fd.c b/lib/stream-fd.c index ad15dca7..2026db6a 100644 --- a/lib/stream-fd.c +++ b/lib/stream-fd.c @@ -28,6 +28,7 @@ #include "leak-checker.h" #include "poll-loop.h" #include "socket-util.h" +#include "stress.h" #include "util.h" #include "stream-provider.h" #include "stream.h" @@ -96,19 +97,39 @@ fd_connect(struct stream *stream) return check_connection_completion(s->fd); } +STRESS_OPTION( + stream_flaky_recv, "simulate failure of fd stream recvs", + 100, 0, -1, 0); + static ssize_t fd_recv(struct stream *stream, void *buffer, size_t n) { struct stream_fd *s = stream_fd_cast(stream); - ssize_t retval = read(s->fd, buffer, n); + ssize_t retval; + + if (STRESS(stream_flaky_recv)) { + return -EIO; + } + + retval = read(s->fd, buffer, n); return retval >= 0 ? retval : -errno; } +STRESS_OPTION( + stream_flaky_send, "simulate failure of fd stream sends", + 100, 0, -1, 0); + static ssize_t fd_send(struct stream *stream, const void *buffer, size_t n) { struct stream_fd *s = stream_fd_cast(stream); - ssize_t retval = write(s->fd, buffer, n); + ssize_t retval; + + if (STRESS(stream_flaky_send)) { + return -EIO; + } + + retval = write(s->fd, buffer, n); return (retval > 0 ? retval : retval == 0 ? -EAGAIN : -errno); diff --git a/lib/stress-unixctl.man b/lib/stress-unixctl.man new file mode 100644 index 00000000..ad265eef --- /dev/null +++ b/lib/stress-unixctl.man @@ -0,0 +1,72 @@ +.SS "STRESS OPTION COMMANDS" +These command manage stress options, which allow developers testing +Open vSwitch to trigger behavior that otherwise would occur only in +corner cases. Developers and testers can thereby more easily discover +bugs that would otherwise manifest only rarely or +nondeterministically. Stress options may cause surprising behavior +even when they do not actually reveal bugs, so they should only be +enabled as part of testing Open vSwitch. +. +.IP "\fBstress/enable\fR" +.IQ "\fBstress/disable\fR" +All stress options are disabled by default. Use \fBstress/enable\fR +to enable stress options and \fBstress/disable\fR to disable them. +. +.IP "\fBstress/list\fR" +Lists and describes the available stress options and their settings in +tabular form. The columns in the table are: +.RS +.IP "NAME" +A single-word identifier for the option, used to identify stress +options to \fBstress/set\fR. +. +.IP "DESCRIPTION" +A description for a person unfamiliar with the detailed internals of +the code what behavior the option affects. +. +.IP "PERIOD" +Currently configured trigger period. If the stress option is +disabled, this is \fBdisabled\fR. Otherwise this is a number giving +the number of occurrences of the event between activations of the +stress option triggers. +. +.IP "MODE" +If the stress option is disabled, this is \fBn/a\fR. Otherwise it is +\fBperiodic\fR if the stress option triggers after exactly the period, +or \fBrandom\fR if it triggers randomly but on average after the +number of occurrences specified by the period. +. +.IP "COUNTER" +If the stress option is disabled, this is \fBn/a\fR. Otherwise it is +the number of occurrences of the event before the next time the stress +option triggers. +. +.IP "HITS" +The number of times that this stress option has triggered since this +program started. +. +.IP "RECOMMENDED" +A suggested period for a person unfamiliar with the internals. It +should put reasonable stress on the system without crippling it. +. +.IP "MINIMUM" +.IQ "MAXIMUM" +Minimum and maximum values allowed for the period. +. +.IP "DEFAULT" +The default period, used when stress options have been enabled (with +\fBstress/enable\fR) but this particular stress option has not been +specifically configured (with \fBstress/set\fR). It is \fBdisabled\fR +if the option is disabled by default. It is nonzero for options that +can be left on at low levels without noticable impact to the end user. +.RE +. +.IP "\fBstress/set \fIoption\fR \fIperiod\fR [\fBrandom\fR|\fBperiodic\fR]" +Sets the period at which stress \fIoption\fR triggers to +\fIperiod\fR. A \fIperiod\fR of 0 disables \fIoption\fR. Specify +\fBrandom\fR to make the option trigger randomly with an average +period of \fIperiod\fR, or \fBperiodic\fR to trigger exactly every +\fIperiod\fR events; the latter is the default. +.IP +If stress options have not been enabled with \fBstress/enable\fR, this +command has no effect. diff --git a/lib/stress.c b/lib/stress.c new file mode 100644 index 00000000..53f6fc02 --- /dev/null +++ b/lib/stress.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2010 Nicira Networks. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "stress.h" +#include +#include +#include "unixctl.h" +#include "dynamic-string.h" +#include "random.h" +#include "util.h" +#include "vlog.h" + +VLOG_DEFINE_THIS_MODULE(stress); + +/* The stress options. */ +#if USE_LINKER_SECTIONS +extern struct stress_option *__start_stress_options[]; +extern struct stress_option *__stop_stress_options[]; +#define stress_options __start_stress_options +#define n_stress_options (__stop_stress_options - __start_stress_options) +#else /* !USE_LINKER_SECTIONS */ +#undef STRESS_OPTION +#define STRESS_OPTION(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT) \ + STRESS_OPTION__(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT); +#include "stress.def" +#undef STRESS_OPTION + +struct stress_option *stress_options[] = { +#define STRESS_OPTION(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT) \ + &stress_##NAME, +#include "stress.def" +#undef STRESS_OPTION +}; +#define n_stress_options ARRAY_SIZE(stress_options) +#endif /* !USE_LINKER_SECTIONS */ + +/* Enable stress options? */ +static bool stress_enabled; + +static void +stress_reset(struct stress_option *option) +{ + if (!option->period || !stress_enabled) { + option->counter = UINT_MAX; + } else if (!option->random) { + option->counter = option->period; + } else if (option->period < UINT32_MAX / 2) { + /* Random distribution with mean of option->period. */ + option->counter = random_uint32() % ((2 * option->period) - 1) + 1; + } else { + option->counter = random_uint32(); + } +} + +static void +stress_enable(bool enable) +{ + if (stress_enabled != enable) { + int i; + + stress_enabled = enable; + for (i = 0; i < n_stress_options; i++) { + stress_reset(stress_options[i]); + } + } +} + +bool +stress_sample_slowpath__(struct stress_option *option) +{ + stress_reset(option); + if (option->period && stress_enabled) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); + + option->hits++; + VLOG_DBG_RL(&rl, "%s hit (%llu total)", option->name, option->hits); + + return true; + } else { + return false; + } +} + +static void +stress_set(struct stress_option *option, unsigned int period, bool random) +{ + if (period > option->max) { + period = option->max; + } + if (period < option->min) { + period = option->min; + } + if (period != option->period || random != option->random) { + option->random = random; + option->period = period; + stress_reset(option); + } +} + +static void +stress_unixctl_list(struct unixctl_conn *conn, const char *args, + void *aux OVS_UNUSED) +{ + int i, found = 0; + struct ds results; + + ds_init(&results); + ds_put_cstr(&results, "NAME (DESCRIPTION)\n"); + ds_put_format(&results, "%11s %10s %10s %10s\n", + "PERIOD", "MODE", "COUNTER", "HITS"); + ds_put_format(&results, "%11s %10s %10s %10s\n", + "RECOMMENDED", "MINIMUM", "MAXIMUM", "DEFAULT"); + for (i = 0; i < n_stress_options; i++) { + struct stress_option *option = stress_options[i]; + if (!*args || strstr(option->name, args)) { + ds_put_format(&results, "\n%s (%s)\n", + option->name, option->description); + if (option->period) { + ds_put_format(&results, "%11u %10s ", option->period, + option->random ? "random" : "periodic"); + if (stress_enabled) { + ds_put_format(&results, "%10u", option->counter); + } else { + ds_put_cstr(&results, " n/a"); + } + } else { + ds_put_format(&results, "%11s %10s %10s", + "disabled", "n/a", "n/a"); + } + ds_put_format(&results, " %10llu\n", option->hits); + ds_put_format(&results, "%11u %10u %10u ", + option->recommended, option->min, option->max); + if (!option->def) { + ds_put_format(&results, "%10s", "disabled"); + } else { + ds_put_format(&results, "%10u", option->def); + } + ds_put_char(&results, '\n'); + found++; + } + } + if (found) { + unixctl_command_reply(conn, 200, ds_cstr(&results)); + } else { + unixctl_command_reply(conn, 404, ""); + } + ds_destroy(&results); +} + +static void +stress_unixctl_enable(struct unixctl_conn *conn, const char *args OVS_UNUSED, + void *aux OVS_UNUSED) +{ + stress_enable(true); + unixctl_command_reply(conn, 200, ""); +} + +static void +stress_unixctl_disable(struct unixctl_conn *conn, const char *args OVS_UNUSED, + void *aux OVS_UNUSED) +{ + stress_enable(false); + unixctl_command_reply(conn, 200, ""); +} + +static void +stress_unixctl_set(struct unixctl_conn *conn, const char *args_, + void *aux OVS_UNUSED) +{ + int code = 404; + char *args = xstrdup(args_); + char *save_ptr = NULL; + char *option_name; + char *option_val; + + option_name = strtok_r(args, " ", &save_ptr); + option_val = strtok_r(NULL, " ", &save_ptr); + if (option_val) { + int i; + for (i = 0; i < n_stress_options; i++) { + struct stress_option *option = stress_options[i]; + if (!strcmp(option_name, option->name)) { + unsigned int period = strtoul(option_val, NULL, 0); + bool random = strstr(args_, "random"); + + stress_set(option, period, random); + code = 200; + break; + } + } + } + unixctl_command_reply(conn, code, ""); + free(args); +} + +/* Exposes ovs-appctl access to the stress options. + * + * This function is not required to simply reference stress options and have + * them fire at their default periods. + */ +void +stress_init_command(void) +{ + unixctl_command_register("stress/list", stress_unixctl_list, NULL); + unixctl_command_register("stress/set", stress_unixctl_set, NULL); + unixctl_command_register("stress/enable", stress_unixctl_enable, NULL); + unixctl_command_register("stress/disable", stress_unixctl_disable, NULL); +} diff --git a/lib/stress.h b/lib/stress.h new file mode 100644 index 00000000..4244fca9 --- /dev/null +++ b/lib/stress.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2010 Nicira Networks. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef STRESS_H +#define STRESS_H 1 + +#include + +struct stress_option { + /* Properties. */ + char *name; /* Short identifier string */ + char *description; /* Description of what the option stresses. */ + unsigned int recommended; /* Recommended period. */ + unsigned int min; /* Minimum period that can be set. */ + unsigned int max; /* Maximum period that can be set. */ + unsigned int def; /* Default value. */ + + /* Configuration. */ + unsigned int period; /* Desired period for firing, 0 to disable. */ + bool random; /* Fire randomly or exactly at period? */ + + /* State. */ + unsigned int counter; /* Number of hits before next firing. */ + unsigned long long int hits; /* Hits since last reset. */ +}; + +/* Creates and initializes a global instance of a stress option. + * + * NAME is a single word descriptive identifier for the option. This is the + * token to pass in to the STRESS() macro at the sites where exectution is to + * be controlled by the option. + * + * DESCRIPTION is a quoted string that should describe to a person unfamiliar + * with the detailed internals of the code what behavior the option affects. + * + * RECOMMENDED is a suggested value for a person unfamiliar with the internals. + * It should put reasonable stress on the system without crippling it. + * + * MIN and MAX are the minimum and maximum values allowed for the option. + * + * DEFAULT is the default value for the option. Specify 0 to disable the + * option by default, which should be the usual choice. But some options can + * be left on at low levels without noticable impact to the end user. An + * example would be failing to allocate a buffer for every 100000th packet + * processed by the system. + */ +#if USE_LINKER_SECTIONS +#define STRESS_OPTION(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT) \ + STRESS_OPTION__(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT); \ + struct stress_option *stress_option_ptr_##NAME \ + __attribute__((section("stress_options"))) = &stress_##NAME +#else +#define STRESS_OPTION(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT) \ + extern struct stress_option stress_##NAME +#endif + +/* Yields true if stress option NAME should be triggered, + * false otherwise. */ +#define STRESS(NAME) stress_sample__(&stress_##NAME) + +void stress_init_command(void); + +/* Implementation details. */ + +#define STRESS_OPTION__(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT) \ + struct stress_option stress_##NAME = \ + { #NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT, \ + DEFAULT ? DEFAULT : 0, /* period */ \ + false, /* random */ \ + UINT_MAX, /* counter */ \ + 0 } /* hits */ + +bool stress_sample_slowpath__(struct stress_option *); +static inline bool stress_sample__(struct stress_option *option) +{ + return --option->counter == 0 && stress_sample_slowpath__(option); +} + +#endif /* STRESS_H */ diff --git a/ofproto/ofproto.c b/ofproto/ofproto.c index c3214c59..bf557cfc 100644 --- a/ofproto/ofproto.c +++ b/ofproto/ofproto.c @@ -63,22 +63,20 @@ VLOG_DEFINE_THIS_MODULE(ofproto); +COVERAGE_DEFINE(facet_changed_rule); +COVERAGE_DEFINE(facet_revalidate); COVERAGE_DEFINE(odp_overflow); -COVERAGE_DEFINE(ofproto_add_wc_flow); COVERAGE_DEFINE(ofproto_agg_request); COVERAGE_DEFINE(ofproto_costly_flags); COVERAGE_DEFINE(ofproto_ctlr_action); -COVERAGE_DEFINE(ofproto_del_wc_flow); -COVERAGE_DEFINE(ofproto_dp_missed); +COVERAGE_DEFINE(ofproto_del_rule); COVERAGE_DEFINE(ofproto_error); COVERAGE_DEFINE(ofproto_expiration); COVERAGE_DEFINE(ofproto_expired); COVERAGE_DEFINE(ofproto_flows_req); COVERAGE_DEFINE(ofproto_flush); COVERAGE_DEFINE(ofproto_invalidated); -COVERAGE_DEFINE(ofproto_mod_wc_flow); COVERAGE_DEFINE(ofproto_no_packet_in); -COVERAGE_DEFINE(ofproto_odp_unchanged); COVERAGE_DEFINE(ofproto_ofconn_stuck); COVERAGE_DEFINE(ofproto_ofp2odp); COVERAGE_DEFINE(ofproto_packet_in); @@ -86,10 +84,6 @@ COVERAGE_DEFINE(ofproto_packet_out); COVERAGE_DEFINE(ofproto_queue_req); COVERAGE_DEFINE(ofproto_recv_openflow); COVERAGE_DEFINE(ofproto_reinit_ports); -COVERAGE_DEFINE(ofproto_revalidate); -COVERAGE_DEFINE(ofproto_revalidate_moved); -COVERAGE_DEFINE(ofproto_revalidate_rule); -COVERAGE_DEFINE(ofproto_subrule_create); COVERAGE_DEFINE(ofproto_unexpected_rule); COVERAGE_DEFINE(ofproto_uninstallable); COVERAGE_DEFINE(ofproto_update_port); diff --git a/ovsdb/ovsdb-server.1.in b/ovsdb/ovsdb-server.1.in index 79a39c66..a89f0919 100644 --- a/ovsdb/ovsdb-server.1.in +++ b/ovsdb/ovsdb-server.1.in @@ -119,6 +119,7 @@ This command might be useful for debugging issues with database clients. . .so lib/vlog-unixctl.man +.so lib/stress-unixctl.man .SH "SEE ALSO" . .BR ovsdb\-tool (1). diff --git a/ovsdb/ovsdb-server.c b/ovsdb/ovsdb-server.c index c7a93815..9f63f3a8 100644 --- a/ovsdb/ovsdb-server.c +++ b/ovsdb/ovsdb-server.c @@ -39,6 +39,7 @@ #include "row.h" #include "stream-ssl.h" #include "stream.h" +#include "stress.h" #include "svec.h" #include "table.h" #include "timeval.h" @@ -87,6 +88,7 @@ main(int argc, char *argv[]) proctitle_init(argc, argv); set_program_name(argv[0]); + stress_init_command(); signal(SIGPIPE, SIG_IGN); process_init(); diff --git a/vswitchd/ovs-vswitchd.8.in b/vswitchd/ovs-vswitchd.8.in index b5c74473..416be246 100644 --- a/vswitchd/ovs-vswitchd.8.in +++ b/vswitchd/ovs-vswitchd.8.in @@ -183,6 +183,7 @@ status of \fIslave\fR changes. Returns the hash value which would be used for \fImac\fR. . .so lib/vlog-unixctl.man +.so lib/stress-unixctl.man .SH "SEE ALSO" .BR ovs\-appctl (8), .BR ovs\-brcompatd (8), diff --git a/vswitchd/ovs-vswitchd.c b/vswitchd/ovs-vswitchd.c index 01f2ac29..93f0371c 100644 --- a/vswitchd/ovs-vswitchd.c +++ b/vswitchd/ovs-vswitchd.c @@ -41,6 +41,7 @@ #include "signals.h" #include "stream-ssl.h" #include "stream.h" +#include "stress.h" #include "svec.h" #include "timeval.h" #include "unixctl.h" @@ -67,6 +68,7 @@ main(int argc, char *argv[]) proctitle_init(argc, argv); set_program_name(argv[0]); + stress_init_command(); remote = parse_options(argc, argv); signal(SIGPIPE, SIG_IGN); sighup = signal_register(SIGHUP);