From 3ed497fc10033c9857140270d60ef6aa2d7c0c08 Mon Sep 17 00:00:00 2001 From: Ben Pfaff Date: Tue, 10 Nov 2009 15:30:49 -0800 Subject: [PATCH] New "reconnect" library for managing network connection attempts. This library implements the reconnection FSM used by the "rconn" library. Therefore, it makes sense to change rconn to use this, and I have a patch to do that, but I am not applying it at the moment to avoid changing unrelated code on the "db" branch. --- lib/automake.mk | 2 + lib/reconnect.c | 523 ++++++++++++++++++++ lib/reconnect.h | 99 ++++ lib/vlog-modules.def | 1 + tests/automake.mk | 5 + tests/reconnect.at | 1039 ++++++++++++++++++++++++++++++++++++++++ tests/test-reconnect.c | 240 ++++++++++ tests/testsuite.at | 1 + 8 files changed, 1910 insertions(+) create mode 100644 lib/reconnect.c create mode 100644 lib/reconnect.h create mode 100644 tests/reconnect.at create mode 100644 tests/test-reconnect.c diff --git a/lib/automake.mk b/lib/automake.mk index 0e93369a..406b4240 100644 --- a/lib/automake.mk +++ b/lib/automake.mk @@ -101,6 +101,8 @@ lib_libopenvswitch_a_SOURCES = \ lib/random.h \ lib/rconn.c \ lib/rconn.h \ + lib/reconnect.c \ + lib/reconnect.h \ lib/rtnetlink.c \ lib/rtnetlink.h \ lib/sat-math.h \ diff --git a/lib/reconnect.c b/lib/reconnect.c new file mode 100644 index 00000000..fadeeb89 --- /dev/null +++ b/lib/reconnect.c @@ -0,0 +1,523 @@ +/* + * Copyright (c) 2008, 2009 Nicira Networks. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "reconnect.h" + +#include +#include + +#include "poll-loop.h" + +#define THIS_MODULE VLM_reconnect +#include "vlog.h" + +#define STATES \ + STATE(VOID, 1 << 0) \ + STATE(BACKOFF, 1 << 1) \ + STATE(CONNECTING, 1 << 2) \ + STATE(ACTIVE, 1 << 3) \ + STATE(IDLE, 1 << 4) \ + STATE(RECONNECT, 1 << 5) +enum state { +#define STATE(NAME, VALUE) S_##NAME = VALUE, + STATES +#undef STATE +}; + +static bool +is_connected_state(enum state state) +{ + return (state & (S_ACTIVE | S_IDLE)) != 0; +} + +struct reconnect { + /* Configuration. */ + char *name; + int min_backoff; + int max_backoff; + int probe_interval; + + /* State. */ + enum state state; + long long int state_entered; + int backoff; + long long int last_received; + long long int last_connected; + + /* These values are simply for statistics reporting, not otherwise used + * directly by anything internal. */ + long long int creation_time; + unsigned int n_attempted_connections, n_successful_connections; + unsigned int total_connected_duration; + unsigned int seqno; +}; + +static void reconnect_transition__(struct reconnect *, long long int now, + enum state state); +static long long int reconnect_deadline__(const struct reconnect *); + +static const char * +reconnect_state_name__(enum state state) +{ + switch (state) { +#define STATE(NAME, VALUE) case S_##NAME: return #NAME; + STATES +#undef STATE + } + return "***ERROR***"; +} + +/* Creates and returns a new reconnect FSM with default settings. The FSM is + * initially disabled. The caller will likely want to call reconnect_enable() + * and reconnect_set_name() on the returned object. */ +struct reconnect * +reconnect_create(long long int now) +{ + struct reconnect *fsm = xzalloc(sizeof *fsm); + + fsm->name = xstrdup("void"); + fsm->min_backoff = 1000; + fsm->max_backoff = 8000; + fsm->probe_interval = 5000; + + fsm->state = S_VOID; + fsm->state_entered = now; + fsm->backoff = 0; + fsm->last_received = now; + fsm->last_connected = now; + fsm->creation_time = now; + + return fsm; +} + +/* Frees 'fsm'. */ +void +reconnect_destroy(struct reconnect *fsm) +{ + if (fsm) { + free(fsm->name); + free(fsm); + } +} + +/* Returns 'fsm''s name. */ +const char * +reconnect_get_name(const struct reconnect *fsm) +{ + return fsm->name; +} + +/* Sets 'fsm''s name to 'name'. If 'name' is null, then "void" is used + * instead. + * + * The name set for 'fsm' is used in log messages. */ +void +reconnect_set_name(struct reconnect *fsm, const char *name) +{ + free(fsm->name); + fsm->name = xstrdup(name ? name : "void"); +} + +/* Return the minimum number of milliseconds to back off between consecutive + * connection attempts. The default is 1000 ms. */ +int +reconnect_get_min_backoff(const struct reconnect *fsm) +{ + return fsm->min_backoff; +} + +/* Return the maximum number of milliseconds to back off between consecutive + * connection attempts. The default is 8000 ms. */ +int +reconnect_get_max_backoff(const struct reconnect *fsm) +{ + return fsm->max_backoff; +} + +/* Returns the "probe interval" for 'fsm' in milliseconds. If this is zero, it + * disables the connection keepalive feature. If it is nonzero, then if the + * interval passes while 'fsm' is connected and without reconnect_received() + * being called for 'fsm', reconnect_run() returns RECONNECT_PROBE. If the + * interval passes again without reconnect_received() being called, + * reconnect_run() returns RECONNECT_DISCONNECT for 'fsm'. */ +int +reconnect_get_probe_interval(const struct reconnect *fsm) +{ + return fsm->probe_interval; +} + +/* Configures the backoff parameters for 'fsm'. 'min_backoff' is the minimum + * number of milliseconds, and 'max_backoff' is the maximum, between connection + * attempts. + * + * 'min_backoff' must be at least 1000, and 'max_backoff' must be greater than + * or equal to 'min_backoff'. */ +void +reconnect_set_backoff(struct reconnect *fsm, int min_backoff, int max_backoff) +{ + fsm->min_backoff = MAX(min_backoff, 1000); + fsm->max_backoff = max_backoff ? MAX(max_backoff, 1000) : 8000; + if (fsm->min_backoff > fsm->max_backoff) { + fsm->max_backoff = fsm->min_backoff; + } + + if (fsm->state == S_BACKOFF && fsm->backoff > max_backoff) { + fsm->backoff = max_backoff; + } +} + +/* Sets the "probe interval" for 'fsm' to 'probe_interval', in milliseconds. + * If this is zero, it disables the connection keepalive feature. If it is + * nonzero, then if the interval passes while 'fsm' is connected and without + * reconnect_received() being called for 'fsm', reconnect_run() returns + * RECONNECT_PROBE. If the interval passes again without reconnect_received() + * being called, reconnect_run() returns RECONNECT_DISCONNECT for 'fsm'. + * + * If 'probe_interval' is nonzero, then it will be forced to a value of at + * least 1000 ms. */ +void +reconnect_set_probe_interval(struct reconnect *fsm, int probe_interval) +{ + fsm->probe_interval = probe_interval ? MAX(1000, probe_interval) : 0; +} + +/* Returns true if 'fsm' has been enabled with reconnect_enable(). Calling + * another function that indicates a change in connection state, such as + * reconnect_disconnected() or reconnect_force_reconnect(), will also enable + * a reconnect FSM. */ +bool +reconnect_is_enabled(const struct reconnect *fsm) +{ + return fsm->state != S_VOID; +} + +/* If 'fsm' is disabled (the default for newly created FSMs), enables it, so + * that the next call to reconnect_run() for 'fsm' will return + * RECONNECT_CONNECT. + * + * If 'fsm' is not disabled, this function has no effect. */ +void +reconnect_enable(struct reconnect *fsm, long long int now) +{ + if (fsm->state == S_VOID) { + reconnect_transition__(fsm, now, S_BACKOFF); + fsm->backoff = 0; + } +} + +/* Disables 'fsm'. Until 'fsm' is enabled again, reconnect_run() will always + * return 0. */ +void +reconnect_disable(struct reconnect *fsm, long long int now) +{ + if (fsm->state != S_VOID) { + reconnect_transition__(fsm, now, S_VOID); + } +} + +/* If 'fsm' is enabled and currently connected (or attempting to connect), + * forces reconnect_run() for 'fsm' to return RECONNECT_DISCONNECT the next + * time it is called, which should cause the client to drop the connection (or + * attempt), back off, and then reconnect. */ +void +reconnect_force_reconnect(struct reconnect *fsm, long long int now) +{ + if (fsm->state & (S_CONNECTING | S_ACTIVE | S_IDLE)) { + reconnect_transition__(fsm, now, S_RECONNECT); + } +} + +/* Tell 'fsm' that the connection dropped or that a connection attempt failed. + * 'error' specifies the reason: a positive value represents an errno value, + * EOF indicates that the connection was closed by the peer (e.g. read() + * returned 0), and 0 indicates no specific error. + * + * The FSM will back off, then reconnect. */ +void +reconnect_disconnected(struct reconnect *fsm, long long int now, int error) +{ + if (fsm->state != S_BACKOFF) { + /* Report what happened. */ + if (fsm->state & (S_ACTIVE | S_IDLE)) { + if (error > 0) { + VLOG_WARN("%s: connection dropped (%s)", + fsm->name, strerror(error)); + } else if (error == EOF) { + VLOG_INFO("%s: connection closed by peer", fsm->name); + } else { + VLOG_INFO("%s: connection dropped", fsm->name); + } + } else { + if (error > 0) { + VLOG_WARN("%s: connection attempt failed (%s)", + fsm->name, strerror(error)); + } else { + VLOG_INFO("%s: connection attempt timed out", fsm->name); + } + } + + /* Back off. */ + if (fsm->state & (S_ACTIVE | S_IDLE) + && fsm->last_received - fsm->last_connected >= fsm->backoff) { + fsm->backoff = fsm->min_backoff; + } else { + if (fsm->backoff < fsm->min_backoff) { + fsm->backoff = fsm->min_backoff; + } else if (fsm->backoff >= fsm->max_backoff / 2) { + fsm->backoff = fsm->max_backoff; + } else { + fsm->backoff *= 2; + } + VLOG_INFO("%s: waiting %.3g seconds before reconnect\n", + fsm->name, fsm->backoff / 1000.0); + } + reconnect_transition__(fsm, now, S_BACKOFF); + } +} + +/* Tell 'fsm' that a connection attempt is in progress. + * + * The FSM will start a timer, after which the connection attempt will be + * aborted (by returning RECONNECT_DISCONNECT from reconect_run()). */ +void +reconnect_connecting(struct reconnect *fsm, long long int now) +{ + if (fsm->state != S_CONNECTING) { + VLOG_INFO("%s: connecting...", fsm->name); + reconnect_transition__(fsm, now, S_CONNECTING); + } +} + +/* Tell 'fsm' that the connection was successful. + * + * The FSM will start the probe interval timer, which is reset by + * reconnect_received(). If the timer expires, a probe will be sent (by + * returning RECONNECT_PROBE from reconnect_run()). If the timer expires + * again without being reset, the connection will be aborted (by returning + * RECONNECT_DISCONNECT from reconnect_run()). */ +void +reconnect_connected(struct reconnect *fsm, long long int now) +{ + if (!is_connected_state(fsm->state)) { + reconnect_connecting(fsm, now); + + VLOG_INFO("%s: connected", fsm->name); + reconnect_transition__(fsm, now, S_ACTIVE); + fsm->last_connected = now; + } +} + +/* Tell 'fsm' that the connection attempt failed. + * + * The FSM will back off and attempt to reconnect. */ +void +reconnect_connect_failed(struct reconnect *fsm, long long int now, int error) +{ + reconnect_connecting(fsm, now); + reconnect_disconnected(fsm, now, error); +} + +/* Tell 'fsm' that some data was received. This resets the probe interval + * timer, so that the connection is known not to be idle. */ +void +reconnect_received(struct reconnect *fsm, long long int now) +{ + if (fsm->state != S_ACTIVE) { + reconnect_transition__(fsm, now, S_ACTIVE); + } + fsm->last_received = now; +} + +static void +reconnect_transition__(struct reconnect *fsm, long long int now, + enum state state) +{ + if (fsm->state == S_CONNECTING) { + fsm->n_attempted_connections++; + if (state == S_ACTIVE) { + fsm->n_successful_connections++; + } + } + if (is_connected_state(fsm->state) != is_connected_state(state)) { + if (is_connected_state(fsm->state)) { + fsm->total_connected_duration += now - fsm->last_connected; + } + fsm->seqno++; + } + + VLOG_DBG("%s: entering %s", fsm->name, reconnect_state_name__(state)); + fsm->state = state; + fsm->state_entered = now; +} + +static long long int +reconnect_deadline__(const struct reconnect *fsm) +{ + assert(fsm->state_entered != LLONG_MIN); + switch (fsm->state) { + case S_VOID: + return LLONG_MAX; + + case S_BACKOFF: + return fsm->state_entered + fsm->backoff; + + case S_CONNECTING: + return fsm->state_entered + MAX(1000, fsm->backoff); + + case S_ACTIVE: + if (fsm->probe_interval) { + long long int base = MAX(fsm->last_received, fsm->state_entered); + return base + fsm->probe_interval; + } + return LLONG_MAX; + + case S_IDLE: + return fsm->state_entered + fsm->probe_interval; + + case S_RECONNECT: + return fsm->state_entered; + } + + NOT_REACHED(); +} + +/* Assesses whether any action should be taken on 'fsm'. The return value is + * one of: + * + * - 0: The client need not take any action. + * + * - RECONNECT_CONNECT: The client should start a connection attempt and + * indicate this by calling reconnect_connecting(). If the connection + * attempt has definitely succeeded, it should call + * reconnect_connected(). If the connection attempt has definitely + * failed, it should call reconnect_connect_failed(). + * + * The FSM is smart enough to back off correctly after successful + * connections that quickly abort, so it is OK to call + * reconnect_connected() after a low-level successful connection + * (e.g. connect()) even if the connection might soon abort due to a + * failure at a high-level (e.g. SSL negotiation failure). + * + * - RECONNECT_DISCONNECT: The client should abort the current connection + * or connection attempt and call reconnect_disconnected() or + * reconnect_connect_failed() to indicate it. + * + * - RECONNECT_PROBE: The client should send some kind of request to the + * peer that will elicit a response, to ensure that the connection is + * indeed in working order. (This will only be returned if the "probe + * interval" is nonzero--see reconnect_set_probe_interval()). + */ +enum reconnect_action +reconnect_run(struct reconnect *fsm, long long int now) +{ + if (now >= reconnect_deadline__(fsm)) { + switch (fsm->state) { + case S_VOID: + return 0; + + case S_BACKOFF: + return RECONNECT_CONNECT; + + case S_CONNECTING: + return RECONNECT_DISCONNECT; + + case S_ACTIVE: + VLOG_DBG("%s: idle %lld ms, sending inactivity probe", fsm->name, + now - MAX(fsm->last_received, fsm->state_entered)); + reconnect_transition__(fsm, now, S_IDLE); + return RECONNECT_PROBE; + + case S_IDLE: + VLOG_ERR("%s: no response to inactivity probe after %.3g " + "seconds, disconnecting", + fsm->name, (now - fsm->state_entered) / 1000.0); + return RECONNECT_DISCONNECT; + + case S_RECONNECT: + return RECONNECT_DISCONNECT; + } + + NOT_REACHED(); + } else { + return fsm->state == S_CONNECTING ? RECONNECT_CONNECT : 0; + } +} + +/* Causes the next call to poll_block() to wake up when reconnect_run() should + * be called on 'fsm'. */ +void +reconnect_wait(struct reconnect *fsm, long long int now) +{ + int timeout = reconnect_timeout(fsm, now); + if (timeout >= 0) { + poll_timer_wait(timeout); + } +} + +/* Returns the number of milliseconds after which reconnect_run() should be + * called on 'fsm' if nothing else notable happens in the meantime, or a + * negative number if this is currently unnecessary. */ +int +reconnect_timeout(struct reconnect *fsm, long long int now) +{ + long long int deadline = reconnect_deadline__(fsm); + if (deadline != LLONG_MAX) { + long long int remaining = deadline - now; + return MAX(0, MIN(INT_MAX, remaining)); + } + return -1; +} + +/* Returns true if 'fsm' is currently believed to be connected, that is, if + * reconnect_connected() was called more recently than any call to + * reconnect_connect_failed() or reconnect_disconnected() or + * reconnect_disable(), and false otherwise. */ +bool +reconnect_is_connected(const struct reconnect *fsm) +{ + return is_connected_state(fsm->state); +} + +/* Returns the number of milliseconds for which 'fsm' has been continuously + * connected to its peer. (If 'fsm' is not currently connected, this is 0.) */ +unsigned int +reconnect_get_connection_duration(const struct reconnect *fsm, + long long int now) +{ + return reconnect_is_connected(fsm) ? now - fsm->last_connected : 0; +} + +/* Copies various statistics for 'fsm' into '*stats'. */ +void +reconnect_get_stats(const struct reconnect *fsm, long long int now, + struct reconnect_stats *stats) +{ + stats->creation_time = fsm->creation_time; + stats->last_received = fsm->last_received; + stats->last_connected = fsm->last_connected; + stats->backoff = fsm->backoff; + stats->seqno = fsm->seqno; + stats->is_connected = reconnect_is_connected(fsm); + stats->current_connection_duration + = reconnect_get_connection_duration(fsm, now); + stats->total_connected_duration = (stats->current_connection_duration + + fsm->total_connected_duration); + stats->n_attempted_connections = fsm->n_attempted_connections; + stats->n_successful_connections = fsm->n_successful_connections; + stats->state = reconnect_state_name__(fsm->state); + stats->state_elapsed = now - fsm->state_entered; +} diff --git a/lib/reconnect.h b/lib/reconnect.h new file mode 100644 index 00000000..3442c07a --- /dev/null +++ b/lib/reconnect.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2009 Nicira Networks. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RECONNECT_H +#define RECONNECT_H 1 + +/* This library implements a finite-state machine for connecting and + * reconnecting to a network resource with exponential backoff. It also + * provides optional support for detecting a connection on which the peer is no + * longer responding. + * + * The library does not implement anything networking related, only an FSM for + * networking code to use. + * + * Many "reconnect" functions take a "now" argument. This makes testing easier + * since there is no hidden state. When not testing, just pass the return + * value of time_msec() from timeval.h. (Perhaps this design should be + * revisited later.) */ + +#include + +struct reconnect *reconnect_create(long long int now); +void reconnect_destroy(struct reconnect *); + +const char *reconnect_get_name(const struct reconnect *); +void reconnect_set_name(struct reconnect *, const char *name); + +int reconnect_get_min_backoff(const struct reconnect *); +int reconnect_get_max_backoff(const struct reconnect *); +int reconnect_get_probe_interval(const struct reconnect *); + +void reconnect_set_backoff(struct reconnect *, + int min_backoff, int max_backoff); +void reconnect_set_probe_interval(struct reconnect *, int probe_interval); + +bool reconnect_is_enabled(const struct reconnect *); +void reconnect_enable(struct reconnect *, long long int now); +void reconnect_disable(struct reconnect *, long long int now); + +void reconnect_force_reconnect(struct reconnect *, long long int now); + +bool reconnect_is_connected(const struct reconnect *); +unsigned int reconnect_get_connection_duration(const struct reconnect *, + long long int now); + +void reconnect_disconnected(struct reconnect *, long long int now, int error); +void reconnect_connecting(struct reconnect *, long long int now); +void reconnect_connected(struct reconnect *, long long int now); +void reconnect_connect_failed(struct reconnect *, long long int now, + int error); +void reconnect_received(struct reconnect *, long long int now); + +enum reconnect_action { + RECONNECT_CONNECT = 1, + RECONNECT_DISCONNECT, + RECONNECT_PROBE, +}; +enum reconnect_action reconnect_run(struct reconnect *, long long int now); +void reconnect_wait(struct reconnect *, long long int now); +int reconnect_timeout(struct reconnect *, long long int now); + +struct reconnect_stats { + /* All times and durations in this structure are in milliseconds. */ + long long int creation_time; /* Time reconnect_create() called. */ + long long int last_received; /* Last call to reconnect_received(). */ + long long int last_connected; /* Last call to reconnect_connected(). */ + int backoff; /* Current backoff duration. */ + + unsigned int seqno; /* # of connections + # of disconnections. */ + + bool is_connected; /* Currently connected? */ + unsigned int current_connection_duration; /* Time of current connection. */ + unsigned int total_connected_duration; /* Sum of all connections. */ + unsigned int n_attempted_connections; + unsigned int n_successful_connections; + + /* These should only be provided to a human user for debugging purposes. + * The client should not attempt to interpret them. */ + const char *state; /* FSM state. */ + unsigned int state_elapsed; /* Time since FSM state entered. */ +}; + +void reconnect_get_stats(const struct reconnect *, long long int now, + struct reconnect_stats *); + +#endif /* reconnect.h */ diff --git a/lib/vlog-modules.def b/lib/vlog-modules.def index 924f556b..5dc77458 100644 --- a/lib/vlog-modules.def +++ b/lib/vlog-modules.def @@ -67,6 +67,7 @@ VLOG_MODULE(port_watcher) VLOG_MODULE(proc_net_compat) VLOG_MODULE(process) VLOG_MODULE(rconn) +VLOG_MODULE(reconnect) VLOG_MODULE(rtnetlink) VLOG_MODULE(stp) VLOG_MODULE(stream_fd) diff --git a/tests/automake.mk b/tests/automake.mk index e8a429fa..f662d950 100644 --- a/tests/automake.mk +++ b/tests/automake.mk @@ -15,6 +15,7 @@ TESTSUITE_AT = \ tests/jsonrpc.at \ tests/timeval.at \ tests/lockfile.at \ + tests/reconnect.at \ tests/ovsdb.at \ tests/ovsdb-file.at \ tests/ovsdb-types.at \ @@ -106,6 +107,10 @@ tests_test_ovsdb_SOURCES = tests/test-ovsdb.c tests_test_ovsdb_LDADD = ovsdb/libovsdb.a lib/libopenvswitch.a EXTRA_DIST += tests/uuidfilt.pl +noinst_PROGRAMS += tests/test-reconnect +tests_test_reconnect_SOURCES = tests/test-reconnect.c +tests_test_reconnect_LDADD = lib/libopenvswitch.a + noinst_PROGRAMS += tests/test-sha1 tests_test_sha1_SOURCES = tests/test-sha1.c tests_test_sha1_LDADD = lib/libopenvswitch.a diff --git a/tests/reconnect.at b/tests/reconnect.at new file mode 100644 index 00000000..33e4b951 --- /dev/null +++ b/tests/reconnect.at @@ -0,0 +1,1039 @@ +AT_BANNER([reconnect library]) + +###################################################################### +AT_SETUP([nothing happens if not enabled]) +AT_KEYWORDS([reconnect]) +AT_DATA([input], [run +timeout +]) +OVS_CHECK_LCOV([test-reconnect < input], [0], + [### t=1000 ### +run +timeout + no timeout +]) +AT_CLEANUP + +###################################################################### +AT_SETUP([quick connect, idle disconnect]) +AT_KEYWORDS([reconnect]) +AT_DATA([input], [enable + +# Connection succeeds. +run +connected + +# Send inactivity probe. +timeout +run + +# Idle timeout kills connection. +timeout +run +disconnected +]) +OVS_CHECK_LCOV([test-reconnect < input], [0], + [### t=1000 ### +enable + in BACKOFF for 0 ms (0 ms backoff) + +# Connection succeeds. +run + should connect +connected + in ACTIVE for 0 ms (0 ms backoff) + 1 successful connections out of 1 attempts, seqno 1 + connected (0 ms), total 0 ms connected + +# Send inactivity probe. +timeout + advance 5000 ms + +### t=6000 ### + in ACTIVE for 5000 ms (0 ms backoff) + connected (5000 ms), total 5000 ms connected +run + should send probe + in IDLE for 0 ms (0 ms backoff) + +# Idle timeout kills connection. +timeout + advance 5000 ms + +### t=11000 ### + in IDLE for 5000 ms (0 ms backoff) + connected (10000 ms), total 10000 ms connected +run + should disconnect +disconnected + in BACKOFF for 0 ms (1000 ms backoff) + 1 successful connections out of 1 attempts, seqno 2 + not connected (0 ms), total 10000 ms connected +]) +AT_CLEANUP + +###################################################################### +AT_SETUP([slow connect, idle disconnect]) +AT_KEYWORDS([reconnect]) +AT_DATA([input], [enable + +# Start connecting. +run +connecting + +# Connect after 500 ms. +advance 500 +run +connected + +# Send inactivity probe. +timeout +run + +# Idle timeout kills connection. +timeout +run +disconnected +]) +OVS_CHECK_LCOV([test-reconnect < input], [0], + [### t=1000 ### +enable + in BACKOFF for 0 ms (0 ms backoff) + +# Start connecting. +run + should connect +connecting + in CONNECTING for 0 ms (0 ms backoff) + +# Connect after 500 ms. +advance 500 + +### t=1500 ### + in CONNECTING for 500 ms (0 ms backoff) +run + should connect +connected + in ACTIVE for 0 ms (0 ms backoff) + created 1000, last received 1000, last connected 1500 + 1 successful connections out of 1 attempts, seqno 1 + connected (0 ms), total 0 ms connected + +# Send inactivity probe. +timeout + advance 5000 ms + +### t=6500 ### + in ACTIVE for 5000 ms (0 ms backoff) + connected (5000 ms), total 5000 ms connected +run + should send probe + in IDLE for 0 ms (0 ms backoff) + +# Idle timeout kills connection. +timeout + advance 5000 ms + +### t=11500 ### + in IDLE for 5000 ms (0 ms backoff) + connected (10000 ms), total 10000 ms connected +run + should disconnect +disconnected + in BACKOFF for 0 ms (1000 ms backoff) + 1 successful connections out of 1 attempts, seqno 2 + not connected (0 ms), total 10000 ms connected +]) +AT_CLEANUP + +###################################################################### +AT_SETUP([connect backs off]) +AT_KEYWORDS([reconnect]) +AT_DATA([input], [enable + +# First connection attempt fails after 1000 ms. +run +connecting +run +timeout +run +connect-failed + +# Back off for 1000 ms. +timeout +run + +# Second connection attempt fails after 1000 ms. +connecting +timeout +run +connect-failed + +# Back off for 2000 ms. +timeout +run + +# Third connection attempt fails after 2000 ms. +connecting +timeout +run +connect-failed + +# Back off for 4000 ms. +timeout +run + +# Third connection attempt fails after 4000 ms. +connecting +timeout +run +connect-failed + +# Back off for 8000 ms. +timeout +run + +# Third connection attempt fails after 8000 ms. +connecting +timeout +run +connect-failed + +# Back off for 8000 ms. +timeout +run + +# Fourth connection attempt fails after 8000 ms. +connecting +timeout +run +connect-failed +]) +OVS_CHECK_LCOV([test-reconnect < input], [0], + [### t=1000 ### +enable + in BACKOFF for 0 ms (0 ms backoff) + +# First connection attempt fails after 1000 ms. +run + should connect +connecting + in CONNECTING for 0 ms (0 ms backoff) +run + should connect +timeout + advance 1000 ms + +### t=2000 ### + in CONNECTING for 1000 ms (0 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (1000 ms backoff) + 0 successful connections out of 1 attempts, seqno 0 + +# Back off for 1000 ms. +timeout + advance 1000 ms + +### t=3000 ### + in BACKOFF for 1000 ms (1000 ms backoff) +run + should connect + +# Second connection attempt fails after 1000 ms. +connecting + in CONNECTING for 0 ms (1000 ms backoff) +timeout + advance 1000 ms + +### t=4000 ### + in CONNECTING for 1000 ms (1000 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (2000 ms backoff) + 0 successful connections out of 2 attempts, seqno 0 + +# Back off for 2000 ms. +timeout + advance 2000 ms + +### t=6000 ### + in BACKOFF for 2000 ms (2000 ms backoff) +run + should connect + +# Third connection attempt fails after 2000 ms. +connecting + in CONNECTING for 0 ms (2000 ms backoff) +timeout + advance 2000 ms + +### t=8000 ### + in CONNECTING for 2000 ms (2000 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (4000 ms backoff) + 0 successful connections out of 3 attempts, seqno 0 + +# Back off for 4000 ms. +timeout + advance 4000 ms + +### t=12000 ### + in BACKOFF for 4000 ms (4000 ms backoff) +run + should connect + +# Third connection attempt fails after 4000 ms. +connecting + in CONNECTING for 0 ms (4000 ms backoff) +timeout + advance 4000 ms + +### t=16000 ### + in CONNECTING for 4000 ms (4000 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (8000 ms backoff) + 0 successful connections out of 4 attempts, seqno 0 + +# Back off for 8000 ms. +timeout + advance 8000 ms + +### t=24000 ### + in BACKOFF for 8000 ms (8000 ms backoff) +run + should connect + +# Third connection attempt fails after 8000 ms. +connecting + in CONNECTING for 0 ms (8000 ms backoff) +timeout + advance 8000 ms + +### t=32000 ### + in CONNECTING for 8000 ms (8000 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (8000 ms backoff) + 0 successful connections out of 5 attempts, seqno 0 + +# Back off for 8000 ms. +timeout + advance 8000 ms + +### t=40000 ### + in BACKOFF for 8000 ms (8000 ms backoff) +run + should connect + +# Fourth connection attempt fails after 8000 ms. +connecting + in CONNECTING for 0 ms (8000 ms backoff) +timeout + advance 8000 ms + +### t=48000 ### + in CONNECTING for 8000 ms (8000 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (8000 ms backoff) + 0 successful connections out of 6 attempts, seqno 0 +]) +AT_CLEANUP + +###################################################################### +AT_SETUP([connections with no data preserve backoff]) +AT_KEYWORDS([reconnect]) +AT_DATA([input], [enable + +# First connect, then idle timeout kills connection. +run +connected +timeout +run +timeout +run +disconnected + +# Back off for 1000 ms. +timeout +run + +# Second connect, then idle timeout kills connection. +run +connected +timeout +run +timeout +run +disconnected + +# Back off for 2000 ms. +timeout +run + +# Third connect, then idle timeout kills connection. +run +connected +timeout +run +timeout +run +disconnected + +# Back off for 4000 ms. +timeout +], [### t=1000 ### +enable + in BACKOFF for 0 ms (0 ms backoff) + +# First connect, then idle timeout kills connection. +run + should connect +connected + in ACTIVE for 0 ms (0 ms backoff) + 1 successful connections out of 1 attempts, seqno 1 + connected (0 ms), total 0 ms connected +timeout + advance 5000 ms + +### t=6000 ### + in ACTIVE for 5000 ms (0 ms backoff) + connected (5000 ms), total 5000 ms connected +run + should send probe + in IDLE for 0 ms (0 ms backoff) +timeout + advance 5000 ms + +### t=11000 ### + in IDLE for 5000 ms (0 ms backoff) + connected (10000 ms), total 10000 ms connected +run + should disconnect +disconnected + in BACKOFF for 0 ms (1000 ms backoff) + 1 successful connections out of 1 attempts, seqno 2 + not connected (0 ms), total 10000 ms connected + +# Back off for 1000 ms. +timeout + advance 1000 ms + +### t=12000 ### + in BACKOFF for 1000 ms (1000 ms backoff) +run + should connect + +# Second connect, then idle timeout kills connection. +run + should connect +connected + in ACTIVE for 0 ms (1000 ms backoff) + created 1000, last received 1000, last connected 12000 + 2 successful connections out of 2 attempts, seqno 3 + connected (0 ms), total 10000 ms connected +timeout + advance 5000 ms + +### t=17000 ### + in ACTIVE for 5000 ms (1000 ms backoff) + connected (5000 ms), total 15000 ms connected +run + should send probe + in IDLE for 0 ms (1000 ms backoff) +timeout + advance 5000 ms + +### t=22000 ### + in IDLE for 5000 ms (1000 ms backoff) + connected (10000 ms), total 20000 ms connected +run + should disconnect +disconnected + in BACKOFF for 0 ms (2000 ms backoff) + 2 successful connections out of 2 attempts, seqno 4 + not connected (0 ms), total 20000 ms connected + +# Back off for 2000 ms. +timeout + advance 2000 ms + +### t=24000 ### + in BACKOFF for 2000 ms (2000 ms backoff) +run + should connect + +# Third connect, then idle timeout kills connection. +run + should connect +connected + in ACTIVE for 0 ms (2000 ms backoff) + created 1000, last received 1000, last connected 24000 + 3 successful connections out of 3 attempts, seqno 5 + connected (0 ms), total 20000 ms connected +timeout + advance 5000 ms + +### t=29000 ### + in ACTIVE for 5000 ms (2000 ms backoff) + connected (5000 ms), total 25000 ms connected +run + should send probe + in IDLE for 0 ms (2000 ms backoff) +timeout + advance 5000 ms + +### t=34000 ### + in IDLE for 5000 ms (2000 ms backoff) + connected (10000 ms), total 30000 ms connected +run + should disconnect +disconnected + in BACKOFF for 0 ms (4000 ms backoff) + 3 successful connections out of 3 attempts, seqno 6 + not connected (0 ms), total 30000 ms connected + +# Back off for 4000 ms. +timeout + advance 4000 ms + +### t=38000 ### + in BACKOFF for 4000 ms (4000 ms backoff) + +]) +AT_CLEANUP + +###################################################################### +AT_SETUP([brief connection preserves backoff]) +AT_KEYWORDS([reconnect]) +AT_DATA([input], [enable + +# First connection attempt fails after 1000 ms. +run +connecting +run +timeout +run +connect-failed + +# Back off for 1000 ms. +timeout +run + +# Second connection attempt fails after 1000 ms. +connecting +timeout +run +connect-failed + +# Back off for 2000 ms. +timeout +run + +# Third connection attempt succeeds after 500 ms. +connecting +advance 500 +run +connected + +# Connection drops after another 250 ms. +advance 250 +disconnected +run + +# Back off for 4000 ms. +timeout +run +]) +OVS_CHECK_LCOV([test-reconnect < input], [0], + [### t=1000 ### +enable + in BACKOFF for 0 ms (0 ms backoff) + +# First connection attempt fails after 1000 ms. +run + should connect +connecting + in CONNECTING for 0 ms (0 ms backoff) +run + should connect +timeout + advance 1000 ms + +### t=2000 ### + in CONNECTING for 1000 ms (0 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (1000 ms backoff) + 0 successful connections out of 1 attempts, seqno 0 + +# Back off for 1000 ms. +timeout + advance 1000 ms + +### t=3000 ### + in BACKOFF for 1000 ms (1000 ms backoff) +run + should connect + +# Second connection attempt fails after 1000 ms. +connecting + in CONNECTING for 0 ms (1000 ms backoff) +timeout + advance 1000 ms + +### t=4000 ### + in CONNECTING for 1000 ms (1000 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (2000 ms backoff) + 0 successful connections out of 2 attempts, seqno 0 + +# Back off for 2000 ms. +timeout + advance 2000 ms + +### t=6000 ### + in BACKOFF for 2000 ms (2000 ms backoff) +run + should connect + +# Third connection attempt succeeds after 500 ms. +connecting + in CONNECTING for 0 ms (2000 ms backoff) +advance 500 + +### t=6500 ### + in CONNECTING for 500 ms (2000 ms backoff) +run + should connect +connected + in ACTIVE for 0 ms (2000 ms backoff) + created 1000, last received 1000, last connected 6500 + 1 successful connections out of 3 attempts, seqno 1 + connected (0 ms), total 0 ms connected + +# Connection drops after another 250 ms. +advance 250 + +### t=6750 ### + in ACTIVE for 250 ms (2000 ms backoff) + connected (250 ms), total 250 ms connected +disconnected + in BACKOFF for 0 ms (4000 ms backoff) + 1 successful connections out of 3 attempts, seqno 2 + not connected (0 ms), total 250 ms connected +run + +# Back off for 4000 ms. +timeout + advance 4000 ms + +### t=10750 ### + in BACKOFF for 4000 ms (4000 ms backoff) +run + should connect +]) +AT_CLEANUP + +###################################################################### +AT_SETUP([brief connection with data preserves backoff]) +AT_KEYWORDS([reconnect]) +AT_DATA([input], [enable + +# First connection attempt fails after 1000 ms. +run +connecting +run +timeout +run +connect-failed + +# Back off for 1000 ms. +timeout +run + +# Second connection attempt fails after 1000 ms. +connecting +timeout +run +connect-failed + +# Back off for 2000 ms. +timeout +run + +# Third connection attempt succeeds after 500 ms. +connecting +advance 500 +run +connected + +# Connection receives 3 chunks of data spaced 250 ms apart. +advance 250 +run +received +advance 250 +run +received +advance 250 +run +received + +# Connection drops. +disconnected +run + +# Back off for 4000 ms. +timeout +run +]) +OVS_CHECK_LCOV([test-reconnect < input], [0], + [### t=1000 ### +enable + in BACKOFF for 0 ms (0 ms backoff) + +# First connection attempt fails after 1000 ms. +run + should connect +connecting + in CONNECTING for 0 ms (0 ms backoff) +run + should connect +timeout + advance 1000 ms + +### t=2000 ### + in CONNECTING for 1000 ms (0 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (1000 ms backoff) + 0 successful connections out of 1 attempts, seqno 0 + +# Back off for 1000 ms. +timeout + advance 1000 ms + +### t=3000 ### + in BACKOFF for 1000 ms (1000 ms backoff) +run + should connect + +# Second connection attempt fails after 1000 ms. +connecting + in CONNECTING for 0 ms (1000 ms backoff) +timeout + advance 1000 ms + +### t=4000 ### + in CONNECTING for 1000 ms (1000 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (2000 ms backoff) + 0 successful connections out of 2 attempts, seqno 0 + +# Back off for 2000 ms. +timeout + advance 2000 ms + +### t=6000 ### + in BACKOFF for 2000 ms (2000 ms backoff) +run + should connect + +# Third connection attempt succeeds after 500 ms. +connecting + in CONNECTING for 0 ms (2000 ms backoff) +advance 500 + +### t=6500 ### + in CONNECTING for 500 ms (2000 ms backoff) +run + should connect +connected + in ACTIVE for 0 ms (2000 ms backoff) + created 1000, last received 1000, last connected 6500 + 1 successful connections out of 3 attempts, seqno 1 + connected (0 ms), total 0 ms connected + +# Connection receives 3 chunks of data spaced 250 ms apart. +advance 250 + +### t=6750 ### + in ACTIVE for 250 ms (2000 ms backoff) + connected (250 ms), total 250 ms connected +run +received + created 1000, last received 6750, last connected 6500 +advance 250 + +### t=7000 ### + in ACTIVE for 500 ms (2000 ms backoff) + connected (500 ms), total 500 ms connected +run +received + created 1000, last received 7000, last connected 6500 +advance 250 + +### t=7250 ### + in ACTIVE for 750 ms (2000 ms backoff) + connected (750 ms), total 750 ms connected +run +received + created 1000, last received 7250, last connected 6500 + +# Connection drops. +disconnected + in BACKOFF for 0 ms (4000 ms backoff) + 1 successful connections out of 3 attempts, seqno 2 + not connected (0 ms), total 750 ms connected +run + +# Back off for 4000 ms. +timeout + advance 4000 ms + +### t=11250 ### + in BACKOFF for 4000 ms (4000 ms backoff) +run + should connect +]) +AT_CLEANUP + +###################################################################### +AT_SETUP([long connection resets backoff]) +AT_KEYWORDS([reconnect]) +AT_DATA([input], [enable + +# First connection attempt fails after 1000 ms. +run +connecting +run +timeout +run +connect-failed + +# Back off for 1000 ms. +timeout +run + +# Second connection attempt fails after 1000 ms. +connecting +timeout +run +connect-failed + +# Back off for 2000 ms. +timeout +run + +# Third connection attempt succeeds after 500 ms. +connecting +advance 500 +run +connected + +# Connection receives 3 chunks of data spaced 2000 ms apart. +advance 2000 +run +received +advance 2000 +run +received +advance 2000 +run +received + +# Connection drops. +disconnected +run + +# Back off for 1000 ms. +timeout +run +]) +OVS_CHECK_LCOV([test-reconnect < input], [0], + [### t=1000 ### +enable + in BACKOFF for 0 ms (0 ms backoff) + +# First connection attempt fails after 1000 ms. +run + should connect +connecting + in CONNECTING for 0 ms (0 ms backoff) +run + should connect +timeout + advance 1000 ms + +### t=2000 ### + in CONNECTING for 1000 ms (0 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (1000 ms backoff) + 0 successful connections out of 1 attempts, seqno 0 + +# Back off for 1000 ms. +timeout + advance 1000 ms + +### t=3000 ### + in BACKOFF for 1000 ms (1000 ms backoff) +run + should connect + +# Second connection attempt fails after 1000 ms. +connecting + in CONNECTING for 0 ms (1000 ms backoff) +timeout + advance 1000 ms + +### t=4000 ### + in CONNECTING for 1000 ms (1000 ms backoff) +run + should disconnect +connect-failed + in BACKOFF for 0 ms (2000 ms backoff) + 0 successful connections out of 2 attempts, seqno 0 + +# Back off for 2000 ms. +timeout + advance 2000 ms + +### t=6000 ### + in BACKOFF for 2000 ms (2000 ms backoff) +run + should connect + +# Third connection attempt succeeds after 500 ms. +connecting + in CONNECTING for 0 ms (2000 ms backoff) +advance 500 + +### t=6500 ### + in CONNECTING for 500 ms (2000 ms backoff) +run + should connect +connected + in ACTIVE for 0 ms (2000 ms backoff) + created 1000, last received 1000, last connected 6500 + 1 successful connections out of 3 attempts, seqno 1 + connected (0 ms), total 0 ms connected + +# Connection receives 3 chunks of data spaced 2000 ms apart. +advance 2000 + +### t=8500 ### + in ACTIVE for 2000 ms (2000 ms backoff) + connected (2000 ms), total 2000 ms connected +run +received + created 1000, last received 8500, last connected 6500 +advance 2000 + +### t=10500 ### + in ACTIVE for 4000 ms (2000 ms backoff) + connected (4000 ms), total 4000 ms connected +run +received + created 1000, last received 10500, last connected 6500 +advance 2000 + +### t=12500 ### + in ACTIVE for 6000 ms (2000 ms backoff) + connected (6000 ms), total 6000 ms connected +run +received + created 1000, last received 12500, last connected 6500 + +# Connection drops. +disconnected + in BACKOFF for 0 ms (1000 ms backoff) + 1 successful connections out of 3 attempts, seqno 2 + not connected (0 ms), total 6000 ms connected +run + +# Back off for 1000 ms. +timeout + advance 1000 ms + +### t=13500 ### + in BACKOFF for 1000 ms (1000 ms backoff) +run + should connect +]) +AT_CLEANUP + +###################################################################### +AT_SETUP([connection attempt fails quickly]) +AT_KEYWORDS([reconnect]) +AT_DATA([input], [enable + +# Connection fails quickly. +run +connect-failed ECONNREFUSED + +# Back off for 1000 ms. +run +timeout + +# Connection fails quickly again. +run +connect-failed ECONNREFUSED + +# Back off for 2000 ms. +run +timeout +]) +OVS_CHECK_LCOV([test-reconnect < input], [0], + [### t=1000 ### +enable + in BACKOFF for 0 ms (0 ms backoff) + +# Connection fails quickly. +run + should connect +connect-failed ECONNREFUSED + in BACKOFF for 0 ms (1000 ms backoff) + 0 successful connections out of 1 attempts, seqno 0 + +# Back off for 1000 ms. +run +timeout + advance 1000 ms + +### t=2000 ### + in BACKOFF for 1000 ms (1000 ms backoff) + +# Connection fails quickly again. +run + should connect +connect-failed ECONNREFUSED + in BACKOFF for 0 ms (2000 ms backoff) + 0 successful connections out of 2 attempts, seqno 0 + +# Back off for 2000 ms. +run +timeout + advance 2000 ms + +### t=4000 ### + in BACKOFF for 2000 ms (2000 ms backoff) +]) +AT_CLEANUP + diff --git a/tests/test-reconnect.c b/tests/test-reconnect.c new file mode 100644 index 00000000..a8784fc8 --- /dev/null +++ b/tests/test-reconnect.c @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2009 Nicira Networks. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "reconnect.h" + +#include +#include +#include +#include + +#include "command-line.h" +#include "compiler.h" +#include "svec.h" +#include "util.h" + +static struct reconnect *reconnect; +static int now; + +static const struct command commands[]; + +static void diff_stats(const struct reconnect_stats *old, + const struct reconnect_stats *new); + +int +main(void) +{ + struct reconnect_stats prev; + int old_time; + char line[128]; + + now = 1000; + reconnect = reconnect_create(now); + reconnect_set_name(reconnect, "remote"); + reconnect_get_stats(reconnect, now, &prev); + printf("### t=%d ###\n", now); + old_time = now; + while (fgets(line, sizeof line, stdin)) { + struct reconnect_stats cur; + struct svec args; + + fputs(line, stdout); + if (line[0] == '#') { + continue; + } + + svec_init(&args); + svec_parse_words(&args, line); + svec_terminate(&args); + if (!svec_is_empty(&args)) { + run_command(args.n, args.names, commands); + } + svec_destroy(&args); + + if (old_time != now) { + printf("\n### t=%d ###\n", now); + old_time = now; + } + + reconnect_get_stats(reconnect, now, &cur); + diff_stats(&prev, &cur); + prev = cur; + } + + return 0; +} + +static void +do_enable(int argc UNUSED, char *argv[] UNUSED) +{ + reconnect_enable(reconnect, now); +} + +static void +do_disable(int argc UNUSED, char *argv[] UNUSED) +{ + reconnect_disable(reconnect, now); +} + +static void +do_force_reconnect(int argc UNUSED, char *argv[] UNUSED) +{ + reconnect_force_reconnect(reconnect, now); +} + +static int +error_from_string(const char *s) +{ + if (!s) { + return 0; + } else if (!strcmp(s, "ECONNREFUSED")) { + return ECONNREFUSED; + } else if (!strcmp(s, "EOF")) { + return EOF; + } else { + ovs_fatal(0, "unknown error '%s'", s); + } +} + +static void +do_disconnected(int argc UNUSED, char *argv[]) +{ + reconnect_disconnected(reconnect, now, error_from_string(argv[1])); +} + +static void +do_connecting(int argc UNUSED, char *argv[] UNUSED) +{ + reconnect_connecting(reconnect, now); +} + +static void +do_connect_failed(int argc UNUSED, char *argv[]) +{ + reconnect_connect_failed(reconnect, now, error_from_string(argv[1])); +} + +static void +do_connected(int argc UNUSED, char *argv[] UNUSED) +{ + reconnect_connected(reconnect, now); +} + +static void +do_received(int argc UNUSED, char *argv[] UNUSED) +{ + reconnect_received(reconnect, now); +} + +static void +do_run(int argc, char *argv[]) +{ + enum reconnect_action action; + + if (argc > 1) { + now += atoi(argv[1]); + } + + action = reconnect_run(reconnect, now); + switch (action) { + default: + if (action != 0) { + NOT_REACHED(); + } + break; + + case RECONNECT_CONNECT: + printf(" should connect\n"); + break; + + case RECONNECT_DISCONNECT: + printf(" should disconnect\n"); + break; + + case RECONNECT_PROBE: + printf(" should send probe\n"); + break; + } +} + +static void +do_advance(int argc UNUSED, char *argv[]) +{ + now += atoi(argv[1]); +} + +static void +do_timeout(int argc UNUSED, char *argv[] UNUSED) +{ + int timeout = reconnect_timeout(reconnect, now); + if (timeout >= 0) { + printf(" advance %d ms\n", timeout); + now += timeout; + } else { + printf(" no timeout\n"); + } +} + +static void +diff_stats(const struct reconnect_stats *old, + const struct reconnect_stats *new) +{ + if (old->state != new->state + || old->state_elapsed != new->state_elapsed + || old->backoff != new->backoff) { + printf(" in %s for %u ms (%d ms backoff)\n", + new->state, new->state_elapsed, new->backoff); + } + if (old->creation_time != new->creation_time + || old->last_received != new->last_received + || old->last_connected != new->last_connected) { + printf(" created %lld, last received %lld, last connected %lld\n", + new->creation_time, new->last_received, new->last_connected); + } + if (old->n_successful_connections != new->n_successful_connections + || old->n_attempted_connections != new->n_attempted_connections + || old->seqno != new->seqno) { + printf(" %u successful connections out of %u attempts, seqno %u\n", + new->n_successful_connections, new->n_attempted_connections, + new->seqno); + } + if (old->is_connected != new->is_connected + || old->current_connection_duration != new->current_connection_duration + || old->total_connected_duration != new->total_connected_duration) { + printf(" %sconnected (%u ms), total %u ms connected\n", + new->is_connected ? "" : "not ", + new->current_connection_duration, + new->total_connected_duration); + } +} + +static const struct command commands[] = { + { "enable", 0, 0, do_enable }, + { "disable", 0, 0, do_disable }, + { "force-reconnect", 0, 0, do_force_reconnect }, + { "disconnected", 0, 1, do_disconnected }, + { "connecting", 0, 0, do_connecting }, + { "connect-failed", 0, 1, do_connect_failed }, + { "connected", 0, 0, do_connected }, + { "received", 0, 0, do_received }, + { "run", 0, 1, do_run }, + { "advance", 1, 1, do_advance }, + { "timeout", 0, 0, do_timeout }, + { NULL, 0, 0, NULL }, +}; + diff --git a/tests/testsuite.at b/tests/testsuite.at index 9b47b8fb..d5eadc21 100644 --- a/tests/testsuite.at +++ b/tests/testsuite.at @@ -27,6 +27,7 @@ m4_include([tests/json.at]) m4_include([tests/jsonrpc.at]) m4_include([tests/timeval.at]) m4_include([tests/lockfile.at]) +m4_include([tests/reconnect.at]) m4_include([tests/ovsdb.at]) m4_include([tests/stp.at]) m4_include([tests/ovs-vsctl.at]) -- 2.30.2