2 * Copyright (c) 2009, 2010, 2011 Nicira Networks.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
26 #include "fail-open.h"
32 #include "poll-loop.h"
40 VLOG_DEFINE_THIS_MODULE(connmgr);
41 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
43 COVERAGE_DEFINE(ofconn_stuck);
45 /* An OpenFlow connection. */
47 struct connmgr *connmgr; /* Connection's manager. */
48 struct list node; /* In struct connmgr's "all_conns" list. */
49 struct rconn *rconn; /* OpenFlow connection. */
50 enum ofconn_type type; /* Type. */
51 enum nx_flow_format flow_format; /* Currently selected flow format. */
53 /* OFPT_PACKET_IN related data. */
54 struct rconn_packet_counter *packet_in_counter; /* # queued on 'rconn'. */
55 #define N_SCHEDULERS 2
56 struct pinsched *schedulers[N_SCHEDULERS];
57 struct pktbuf *pktbuf; /* OpenFlow packet buffers. */
58 int miss_send_len; /* Bytes to send of buffered packets. */
60 /* Number of OpenFlow messages queued on 'rconn' as replies to OpenFlow
61 * requests, and the maximum number before we stop reading OpenFlow
63 #define OFCONN_REPLY_MAX 100
64 struct rconn_packet_counter *reply_counter;
66 /* type == OFCONN_PRIMARY only. */
67 enum nx_role role; /* Role. */
68 struct hmap_node hmap_node; /* In struct connmgr's "controllers" map. */
69 enum ofproto_band band; /* In-band or out-of-band? */
72 static struct ofconn *ofconn_create(struct connmgr *, struct rconn *,
74 static void ofconn_destroy(struct ofconn *);
76 static void ofconn_reconfigure(struct ofconn *,
77 const struct ofproto_controller *);
79 static void ofconn_run(struct ofconn *,
80 void (*handle_openflow)(struct ofconn *,
81 struct ofpbuf *ofp_msg));
82 static void ofconn_wait(struct ofconn *);
84 static const char *ofconn_get_target(const struct ofconn *);
85 static char *ofconn_make_name(const struct connmgr *, const char *target);
87 static void ofconn_set_rate_limit(struct ofconn *, int rate, int burst);
89 static bool ofconn_receives_async_msgs(const struct ofconn *);
91 static void ofconn_send(const struct ofconn *, struct ofpbuf *,
92 struct rconn_packet_counter *);
94 static void do_send_packet_in(struct ofpbuf *, void *ofconn_);
96 /* A listener for incoming OpenFlow "service" connections. */
98 struct hmap_node node; /* In struct connmgr's "services" hmap. */
99 struct pvconn *pvconn; /* OpenFlow connection listener. */
101 /* These are not used by ofservice directly. They are settings for
102 * accepted "struct ofconn"s from the pvconn. */
103 int probe_interval; /* Max idle time before probing, in seconds. */
104 int rate_limit; /* Max packet-in rate in packets per second. */
105 int burst_limit; /* Limit on accumulating packet credits. */
108 static void ofservice_reconfigure(struct ofservice *,
109 const struct ofproto_controller *);
110 static int ofservice_create(struct connmgr *, const char *target);
111 static void ofservice_destroy(struct connmgr *, struct ofservice *);
112 static struct ofservice *ofservice_lookup(struct connmgr *,
115 /* Connection manager for an OpenFlow switch. */
117 struct ofproto *ofproto;
119 char *local_port_name;
121 /* OpenFlow connections. */
122 struct hmap controllers; /* Controller "struct ofconn"s. */
123 struct list all_conns; /* Contains "struct ofconn"s. */
125 /* OpenFlow listeners. */
126 struct hmap services; /* Contains "struct ofservice"s. */
127 struct pvconn **snoops;
131 struct fail_open *fail_open;
132 enum ofproto_fail_mode fail_mode;
134 /* In-band control. */
135 struct in_band *in_band;
136 long long int next_in_band_update;
137 struct sockaddr_in *extra_in_band_remotes;
138 size_t n_extra_remotes;
142 static void update_in_band_remotes(struct connmgr *);
143 static void add_snooper(struct connmgr *, struct vconn *);
145 /* Creates and returns a new connection manager owned by 'ofproto'. 'name' is
146 * a name for the ofproto suitable for using in log messages.
147 * 'local_port_name' is the name of the local port (OFPP_LOCAL) within
150 connmgr_create(struct ofproto *ofproto,
151 const char *name, const char *local_port_name)
155 mgr = xmalloc(sizeof *mgr);
156 mgr->ofproto = ofproto;
157 mgr->name = xstrdup(name);
158 mgr->local_port_name = xstrdup(local_port_name);
160 hmap_init(&mgr->controllers);
161 list_init(&mgr->all_conns);
163 hmap_init(&mgr->services);
167 mgr->fail_open = NULL;
168 mgr->fail_mode = OFPROTO_FAIL_SECURE;
171 mgr->next_in_band_update = LLONG_MAX;
172 mgr->extra_in_band_remotes = NULL;
173 mgr->n_extra_remotes = 0;
174 mgr->in_band_queue = -1;
179 /* Frees 'mgr' and all of its resources. */
181 connmgr_destroy(struct connmgr *mgr)
183 struct ofservice *ofservice, *next_ofservice;
184 struct ofconn *ofconn, *next_ofconn;
191 LIST_FOR_EACH_SAFE (ofconn, next_ofconn, node, &mgr->all_conns) {
192 ofconn_destroy(ofconn);
194 hmap_destroy(&mgr->controllers);
196 HMAP_FOR_EACH_SAFE (ofservice, next_ofservice, node, &mgr->services) {
197 ofservice_destroy(mgr, ofservice);
199 hmap_destroy(&mgr->services);
201 for (i = 0; i < mgr->n_snoops; i++) {
202 pvconn_close(mgr->snoops[i]);
206 fail_open_destroy(mgr->fail_open);
207 mgr->fail_open = NULL;
209 in_band_destroy(mgr->in_band);
211 free(mgr->extra_in_band_remotes);
213 free(mgr->local_port_name);
218 /* Does all of the periodic maintenance required by 'mgr'. Calls
219 * 'handle_openflow' for each message received on an OpenFlow connection,
220 * passing along the OpenFlow connection itself and the message that was sent.
221 * The 'handle_openflow' callback must not free the message. */
223 connmgr_run(struct connmgr *mgr,
224 void (*handle_openflow)(struct ofconn *, struct ofpbuf *ofp_msg))
226 struct ofconn *ofconn, *next_ofconn;
227 struct ofservice *ofservice;
231 if (time_msec() >= mgr->next_in_band_update) {
232 update_in_band_remotes(mgr);
234 in_band_run(mgr->in_band);
237 LIST_FOR_EACH_SAFE (ofconn, next_ofconn, node, &mgr->all_conns) {
238 ofconn_run(ofconn, handle_openflow);
241 /* Fail-open maintenance. Do this after processing the ofconns since
242 * fail-open checks the status of the controller rconn. */
243 if (mgr->fail_open) {
244 fail_open_run(mgr->fail_open);
247 HMAP_FOR_EACH (ofservice, node, &mgr->services) {
251 retval = pvconn_accept(ofservice->pvconn, OFP_VERSION, &vconn);
256 rconn = rconn_create(ofservice->probe_interval, 0);
257 name = ofconn_make_name(mgr, vconn_get_name(vconn));
258 rconn_connect_unreliably(rconn, vconn, name);
261 ofconn = ofconn_create(mgr, rconn, OFCONN_SERVICE);
262 ofconn_set_rate_limit(ofconn, ofservice->rate_limit,
263 ofservice->burst_limit);
264 } else if (retval != EAGAIN) {
265 VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
269 for (i = 0; i < mgr->n_snoops; i++) {
273 retval = pvconn_accept(mgr->snoops[i], OFP_VERSION, &vconn);
275 add_snooper(mgr, vconn);
276 } else if (retval != EAGAIN) {
277 VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
282 /* Causes the poll loop to wake up when connmgr_run() needs to run. */
284 connmgr_wait(struct connmgr *mgr)
286 struct ofservice *ofservice;
287 struct ofconn *ofconn;
290 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
294 poll_timer_wait_until(mgr->next_in_band_update);
295 in_band_wait(mgr->in_band);
297 if (mgr->fail_open) {
298 fail_open_wait(mgr->fail_open);
300 HMAP_FOR_EACH (ofservice, node, &mgr->services) {
301 pvconn_wait(ofservice->pvconn);
303 for (i = 0; i < mgr->n_snoops; i++) {
304 pvconn_wait(mgr->snoops[i]);
308 /* Returns the ofproto that owns 'ofconn''s connmgr. */
310 ofconn_get_ofproto(const struct ofconn *ofconn)
312 return ofconn->connmgr->ofproto;
315 /* OpenFlow configuration. */
317 static void add_controller(struct connmgr *, const char *target);
318 static struct ofconn *find_controller_by_target(struct connmgr *,
320 static void update_fail_open(struct connmgr *);
321 static int set_pvconns(struct pvconn ***pvconnsp, size_t *n_pvconnsp,
322 const struct sset *);
324 /* Returns true if 'mgr' has any configured primary controllers.
326 * Service controllers do not count, but configured primary controllers do
327 * count whether or not they are currently connected. */
329 connmgr_has_controllers(const struct connmgr *mgr)
331 return !hmap_is_empty(&mgr->controllers);
334 /* Initializes 'info' and populates it with information about each configured
335 * primary controller. The keys in 'info' are the controllers' targets; the
336 * data values are corresponding "struct ofproto_controller_info".
338 * The caller owns 'info' and everything in it and should free it when it is no
341 connmgr_get_controller_info(struct connmgr *mgr, struct shash *info)
343 const struct ofconn *ofconn;
347 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
348 const struct rconn *rconn = ofconn->rconn;
349 time_t now = time_now();
350 time_t last_connection = rconn_get_last_connection(rconn);
351 time_t last_disconnect = rconn_get_last_disconnect(rconn);
352 int last_error = rconn_get_last_error(rconn);
353 struct ofproto_controller_info *cinfo = xmalloc(sizeof *cinfo);
355 shash_add(info, rconn_get_target(rconn), cinfo);
357 cinfo->is_connected = rconn_is_connected(rconn);
358 cinfo->role = ofconn->role;
363 cinfo->pairs.keys[cinfo->pairs.n] = "last_error";
364 cinfo->pairs.values[cinfo->pairs.n++] =
365 xstrdup(ovs_retval_to_string(last_error));
368 cinfo->pairs.keys[cinfo->pairs.n] = "state";
369 cinfo->pairs.values[cinfo->pairs.n++] =
370 xstrdup(rconn_get_state(rconn));
372 if (last_connection != TIME_MIN) {
373 cinfo->pairs.keys[cinfo->pairs.n] = "sec_since_connect";
374 cinfo->pairs.values[cinfo->pairs.n++]
375 = xasprintf("%ld", (long int) (now - last_connection));
378 if (last_disconnect != TIME_MIN) {
379 cinfo->pairs.keys[cinfo->pairs.n] = "sec_since_disconnect";
380 cinfo->pairs.values[cinfo->pairs.n++]
381 = xasprintf("%ld", (long int) (now - last_disconnect));
386 /* Changes 'mgr''s set of controllers to the 'n_controllers' controllers in
389 connmgr_set_controllers(struct connmgr *mgr,
390 const struct ofproto_controller *controllers,
391 size_t n_controllers)
393 struct shash new_controllers;
394 struct ofconn *ofconn, *next_ofconn;
395 struct ofservice *ofservice, *next_ofservice;
398 /* Create newly configured controllers and services.
399 * Create a name to ofproto_controller mapping in 'new_controllers'. */
400 shash_init(&new_controllers);
401 for (i = 0; i < n_controllers; i++) {
402 const struct ofproto_controller *c = &controllers[i];
404 if (!vconn_verify_name(c->target)) {
405 if (!find_controller_by_target(mgr, c->target)) {
406 add_controller(mgr, c->target);
408 } else if (!pvconn_verify_name(c->target)) {
409 if (!ofservice_lookup(mgr, c->target)) {
410 ofservice_create(mgr, c->target);
413 VLOG_WARN_RL(&rl, "%s: unsupported controller \"%s\"",
414 mgr->name, c->target);
418 shash_add_once(&new_controllers, c->target, &controllers[i]);
421 /* Delete controllers that are no longer configured.
422 * Update configuration of all now-existing controllers. */
423 HMAP_FOR_EACH_SAFE (ofconn, next_ofconn, hmap_node, &mgr->controllers) {
424 struct ofproto_controller *c;
426 c = shash_find_data(&new_controllers, ofconn_get_target(ofconn));
428 ofconn_destroy(ofconn);
430 ofconn_reconfigure(ofconn, c);
434 /* Delete services that are no longer configured.
435 * Update configuration of all now-existing services. */
436 HMAP_FOR_EACH_SAFE (ofservice, next_ofservice, node, &mgr->services) {
437 struct ofproto_controller *c;
439 c = shash_find_data(&new_controllers,
440 pvconn_get_name(ofservice->pvconn));
442 ofservice_destroy(mgr, ofservice);
444 ofservice_reconfigure(ofservice, c);
448 shash_destroy(&new_controllers);
450 update_in_band_remotes(mgr);
451 update_fail_open(mgr);
454 /* Drops the connections between 'mgr' and all of its primary and secondary
455 * controllers, forcing them to reconnect. */
457 connmgr_reconnect(const struct connmgr *mgr)
459 struct ofconn *ofconn;
461 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
462 rconn_reconnect(ofconn->rconn);
466 /* Sets the "snoops" for 'mgr' to the pvconn targets listed in 'snoops'.
468 * A "snoop" is a pvconn to which every OpenFlow message to or from the most
469 * important controller on 'mgr' is mirrored. */
471 connmgr_set_snoops(struct connmgr *mgr, const struct sset *snoops)
473 return set_pvconns(&mgr->snoops, &mgr->n_snoops, snoops);
476 /* Adds each of the snoops currently configured on 'mgr' to 'snoops'. */
478 connmgr_get_snoops(const struct connmgr *mgr, struct sset *snoops)
482 for (i = 0; i < mgr->n_snoops; i++) {
483 sset_add(snoops, pvconn_get_name(mgr->snoops[i]));
487 /* Returns true if 'mgr' has at least one snoop, false if it has none. */
489 connmgr_has_snoops(const struct connmgr *mgr)
491 return mgr->n_snoops > 0;
494 /* Creates a new controller for 'target' in 'mgr'. update_controller() needs
495 * to be called later to finish the new ofconn's configuration. */
497 add_controller(struct connmgr *mgr, const char *target)
499 char *name = ofconn_make_name(mgr, target);
500 struct ofconn *ofconn;
502 ofconn = ofconn_create(mgr, rconn_create(5, 8), OFCONN_PRIMARY);
503 ofconn->pktbuf = pktbuf_create();
504 ofconn->miss_send_len = OFP_DEFAULT_MISS_SEND_LEN;
505 rconn_connect(ofconn->rconn, target, name);
506 hmap_insert(&mgr->controllers, &ofconn->hmap_node, hash_string(target, 0));
511 static struct ofconn *
512 find_controller_by_target(struct connmgr *mgr, const char *target)
514 struct ofconn *ofconn;
516 HMAP_FOR_EACH_WITH_HASH (ofconn, hmap_node,
517 hash_string(target, 0), &mgr->controllers) {
518 if (!strcmp(ofconn_get_target(ofconn), target)) {
526 update_in_band_remotes(struct connmgr *mgr)
528 struct sockaddr_in *addrs;
529 size_t max_addrs, n_addrs;
530 struct ofconn *ofconn;
533 /* Allocate enough memory for as many remotes as we could possibly have. */
534 max_addrs = mgr->n_extra_remotes + hmap_count(&mgr->controllers);
535 addrs = xmalloc(max_addrs * sizeof *addrs);
538 /* Add all the remotes. */
539 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
540 struct sockaddr_in *sin = &addrs[n_addrs];
542 if (ofconn->band == OFPROTO_OUT_OF_BAND) {
546 sin->sin_addr.s_addr = rconn_get_remote_ip(ofconn->rconn);
547 if (sin->sin_addr.s_addr) {
548 sin->sin_port = rconn_get_remote_port(ofconn->rconn);
552 for (i = 0; i < mgr->n_extra_remotes; i++) {
553 addrs[n_addrs++] = mgr->extra_in_band_remotes[i];
556 /* Create or update or destroy in-band. */
559 in_band_create(mgr->ofproto, mgr->local_port_name, &mgr->in_band);
562 in_band_set_remotes(mgr->in_band, addrs, n_addrs);
564 in_band_set_queue(mgr->in_band, mgr->in_band_queue);
565 mgr->next_in_band_update = time_msec() + 1000;
567 in_band_destroy(mgr->in_band);
576 update_fail_open(struct connmgr *mgr)
578 if (connmgr_has_controllers(mgr)
579 && mgr->fail_mode == OFPROTO_FAIL_STANDALONE) {
580 if (!mgr->fail_open) {
581 mgr->fail_open = fail_open_create(mgr->ofproto, mgr);
584 fail_open_destroy(mgr->fail_open);
585 mgr->fail_open = NULL;
590 set_pvconns(struct pvconn ***pvconnsp, size_t *n_pvconnsp,
591 const struct sset *sset)
593 struct pvconn **pvconns = *pvconnsp;
594 size_t n_pvconns = *n_pvconnsp;
599 for (i = 0; i < n_pvconns; i++) {
600 pvconn_close(pvconns[i]);
604 pvconns = xmalloc(sset_count(sset) * sizeof *pvconns);
606 SSET_FOR_EACH (name, sset) {
607 struct pvconn *pvconn;
610 error = pvconn_open(name, &pvconn);
612 pvconns[n_pvconns++] = pvconn;
614 VLOG_ERR("failed to listen on %s: %s", name, strerror(error));
622 *n_pvconnsp = n_pvconns;
627 /* Returns a "preference level" for snooping 'ofconn'. A higher return value
628 * means that 'ofconn' is more interesting for monitoring than a lower return
631 snoop_preference(const struct ofconn *ofconn)
633 switch (ofconn->role) {
641 /* Shouldn't happen. */
646 /* One of 'mgr''s "snoop" pvconns has accepted a new connection on 'vconn'.
647 * Connects this vconn to a controller. */
649 add_snooper(struct connmgr *mgr, struct vconn *vconn)
651 struct ofconn *ofconn, *best;
653 /* Pick a controller for monitoring. */
655 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
656 if (ofconn->type == OFCONN_PRIMARY
657 && (!best || snoop_preference(ofconn) > snoop_preference(best))) {
663 rconn_add_monitor(best->rconn, vconn);
665 VLOG_INFO_RL(&rl, "no controller connection to snoop");
670 /* Public ofconn functions. */
672 /* Returns the connection type, either OFCONN_PRIMARY or OFCONN_SERVICE. */
674 ofconn_get_type(const struct ofconn *ofconn)
679 /* Returns the role configured for 'ofconn'.
681 * The default role, if no other role has been set, is NX_ROLE_OTHER. */
683 ofconn_get_role(const struct ofconn *ofconn)
688 /* Changes 'ofconn''s role to 'role'. If 'role' is NX_ROLE_MASTER then any
689 * existing master is demoted to a slave. */
691 ofconn_set_role(struct ofconn *ofconn, enum nx_role role)
693 if (role == NX_ROLE_MASTER) {
694 struct ofconn *other;
696 HMAP_FOR_EACH (other, hmap_node, &ofconn->connmgr->controllers) {
697 if (other->role == NX_ROLE_MASTER) {
698 other->role = NX_ROLE_SLAVE;
705 /* Returns the currently configured flow format for 'ofconn', one of NXFF_*.
707 * The default, if no other format has been set, is NXFF_OPENFLOW10. */
709 ofconn_get_flow_format(struct ofconn *ofconn)
711 return ofconn->flow_format;
714 /* Sets the flow format for 'ofconn' to 'flow_format' (one of NXFF_*). */
716 ofconn_set_flow_format(struct ofconn *ofconn, enum nx_flow_format flow_format)
718 ofconn->flow_format = flow_format;
721 /* Returns the default miss send length for 'ofconn'. */
723 ofconn_get_miss_send_len(const struct ofconn *ofconn)
725 return ofconn->miss_send_len;
728 /* Sets the default miss send length for 'ofconn' to 'miss_send_len'. */
730 ofconn_set_miss_send_len(struct ofconn *ofconn, int miss_send_len)
732 ofconn->miss_send_len = miss_send_len;
735 /* Sends 'msg' on 'ofconn', accounting it as a reply. (If there is a
736 * sufficient number of OpenFlow replies in-flight on a single ofconn, then the
737 * connmgr will stop accepting new OpenFlow requests on that ofconn until the
738 * controller has accepted some of the replies.) */
740 ofconn_send_reply(const struct ofconn *ofconn, struct ofpbuf *msg)
742 ofconn_send(ofconn, msg, ofconn->reply_counter);
745 /* Same as pktbuf_retrieve(), using the pktbuf owned by 'ofconn'. */
747 ofconn_pktbuf_retrieve(struct ofconn *ofconn, uint32_t id,
748 struct ofpbuf **bufferp, uint16_t *in_port)
750 return pktbuf_retrieve(ofconn->pktbuf, id, bufferp, in_port);
753 /* Private ofconn functions. */
756 ofconn_get_target(const struct ofconn *ofconn)
758 return rconn_get_target(ofconn->rconn);
761 static struct ofconn *
762 ofconn_create(struct connmgr *mgr, struct rconn *rconn, enum ofconn_type type)
764 struct ofconn *ofconn = xzalloc(sizeof *ofconn);
765 ofconn->connmgr = mgr;
766 list_push_back(&mgr->all_conns, &ofconn->node);
767 ofconn->rconn = rconn;
769 ofconn->flow_format = NXFF_OPENFLOW10;
770 ofconn->role = NX_ROLE_OTHER;
771 ofconn->packet_in_counter = rconn_packet_counter_create ();
772 ofconn->pktbuf = NULL;
773 ofconn->miss_send_len = 0;
774 ofconn->reply_counter = rconn_packet_counter_create ();
779 ofconn_destroy(struct ofconn *ofconn)
781 if (ofconn->type == OFCONN_PRIMARY) {
782 hmap_remove(&ofconn->connmgr->controllers, &ofconn->hmap_node);
785 list_remove(&ofconn->node);
786 rconn_destroy(ofconn->rconn);
787 rconn_packet_counter_destroy(ofconn->packet_in_counter);
788 rconn_packet_counter_destroy(ofconn->reply_counter);
789 pktbuf_destroy(ofconn->pktbuf);
793 /* Reconfigures 'ofconn' to match 'c'. 'ofconn' and 'c' must have the same
796 ofconn_reconfigure(struct ofconn *ofconn, const struct ofproto_controller *c)
800 ofconn->band = c->band;
802 rconn_set_max_backoff(ofconn->rconn, c->max_backoff);
804 probe_interval = c->probe_interval ? MAX(c->probe_interval, 5) : 0;
805 rconn_set_probe_interval(ofconn->rconn, probe_interval);
807 ofconn_set_rate_limit(ofconn, c->rate_limit, c->burst_limit);
811 ofconn_run(struct ofconn *ofconn,
812 void (*handle_openflow)(struct ofconn *, struct ofpbuf *ofp_msg))
814 struct connmgr *mgr = ofconn->connmgr;
818 for (i = 0; i < N_SCHEDULERS; i++) {
819 pinsched_run(ofconn->schedulers[i], do_send_packet_in, ofconn);
822 rconn_run(ofconn->rconn);
824 if (rconn_packet_counter_read (ofconn->reply_counter) < OFCONN_REPLY_MAX) {
825 /* Limit the number of iterations to prevent other tasks from
827 for (iteration = 0; iteration < 50; iteration++) {
828 struct ofpbuf *of_msg = rconn_recv(ofconn->rconn);
832 if (mgr->fail_open) {
833 fail_open_maybe_recover(mgr->fail_open);
835 handle_openflow(ofconn, of_msg);
836 ofpbuf_delete(of_msg);
840 if (!rconn_is_alive(ofconn->rconn)) {
841 ofconn_destroy(ofconn);
846 ofconn_wait(struct ofconn *ofconn)
850 for (i = 0; i < N_SCHEDULERS; i++) {
851 pinsched_wait(ofconn->schedulers[i]);
853 rconn_run_wait(ofconn->rconn);
854 if (rconn_packet_counter_read (ofconn->reply_counter) < OFCONN_REPLY_MAX) {
855 rconn_recv_wait(ofconn->rconn);
857 COVERAGE_INC(ofconn_stuck);
861 /* Returns true if 'ofconn' should receive asynchronous messages. */
863 ofconn_receives_async_msgs(const struct ofconn *ofconn)
865 if (!rconn_is_connected(ofconn->rconn)) {
867 } else if (ofconn->type == OFCONN_PRIMARY) {
868 /* Primary controllers always get asynchronous messages unless they
869 * have configured themselves as "slaves". */
870 return ofconn->role != NX_ROLE_SLAVE;
872 /* Service connections don't get asynchronous messages unless they have
873 * explicitly asked for them by setting a nonzero miss send length. */
874 return ofconn->miss_send_len > 0;
878 /* Returns a human-readable name for an OpenFlow connection between 'mgr' and
879 * 'target', suitable for use in log messages for identifying the connection.
881 * The name is dynamically allocated. The caller should free it (with free())
882 * when it is no longer needed. */
884 ofconn_make_name(const struct connmgr *mgr, const char *target)
886 return xasprintf("%s<->%s", mgr->name, target);
890 ofconn_set_rate_limit(struct ofconn *ofconn, int rate, int burst)
894 for (i = 0; i < N_SCHEDULERS; i++) {
895 struct pinsched **s = &ofconn->schedulers[i];
899 *s = pinsched_create(rate, burst);
901 pinsched_set_limits(*s, rate, burst);
904 pinsched_destroy(*s);
911 ofconn_send(const struct ofconn *ofconn, struct ofpbuf *msg,
912 struct rconn_packet_counter *counter)
914 update_openflow_length(msg);
915 if (rconn_send(ofconn->rconn, msg, counter)) {
920 /* Sending asynchronous messages. */
922 static void schedule_packet_in(struct ofconn *, const struct dpif_upcall *,
923 const struct flow *, struct ofpbuf *rw_packet);
925 /* Sends an OFPT_PORT_STATUS message with 'opp' and 'reason' to appropriate
926 * controllers managed by 'mgr'. */
928 connmgr_send_port_status(struct connmgr *mgr, const struct ofp_phy_port *opp,
931 /* XXX Should limit the number of queued port status change messages. */
932 struct ofconn *ofconn;
934 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
935 struct ofp_port_status *ops;
938 /* Primary controllers, even slaves, should always get port status
939 updates. Otherwise obey ofconn_receives_async_msgs(). */
940 if (ofconn->type != OFCONN_PRIMARY
941 && !ofconn_receives_async_msgs(ofconn)) {
945 ops = make_openflow_xid(sizeof *ops, OFPT_PORT_STATUS, 0, &b);
946 ops->reason = reason;
948 ofconn_send(ofconn, b, NULL);
952 /* Sends an OFPT_FLOW_REMOVED or NXT_FLOW_REMOVED message based on 'fr' to
953 * appropriate controllers managed by 'mgr'. */
955 connmgr_send_flow_removed(struct connmgr *mgr,
956 const struct ofputil_flow_removed *fr)
958 struct ofconn *ofconn;
960 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
963 if (!ofconn_receives_async_msgs(ofconn)) {
967 /* Account flow expirations as replies to OpenFlow requests. That
968 * works because preventing OpenFlow requests from being processed also
969 * prevents new flows from being added (and expiring). (It also
970 * prevents processing OpenFlow requests that would not add new flows,
971 * so it is imperfect.) */
972 msg = ofputil_encode_flow_removed(fr, ofconn->flow_format);
973 ofconn_send_reply(ofconn, msg);
977 /* Given 'upcall', of type DPIF_UC_ACTION or DPIF_UC_MISS, sends an
978 * OFPT_PACKET_IN message to each OpenFlow controller as necessary according to
979 * their individual configurations.
981 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
982 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
983 * of 'rw_packet' is transferred to this function. */
985 connmgr_send_packet_in(struct connmgr *mgr, const struct dpif_upcall *upcall,
986 const struct flow *flow, struct ofpbuf *rw_packet)
988 struct ofconn *ofconn, *prev;
991 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
992 if (ofconn_receives_async_msgs(ofconn)) {
994 schedule_packet_in(prev, upcall, flow, NULL);
1000 schedule_packet_in(prev, upcall, flow, rw_packet);
1002 ofpbuf_delete(rw_packet);
1006 /* pinsched callback for sending 'ofp_packet_in' on 'ofconn'. */
1008 do_send_packet_in(struct ofpbuf *ofp_packet_in, void *ofconn_)
1010 struct ofconn *ofconn = ofconn_;
1012 rconn_send_with_limit(ofconn->rconn, ofp_packet_in,
1013 ofconn->packet_in_counter, 100);
1016 /* Takes 'upcall', whose packet has the flow specified by 'flow', composes an
1017 * OpenFlow packet-in message from it, and passes it to 'ofconn''s packet
1018 * scheduler for sending.
1020 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
1021 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
1022 * of 'rw_packet' is transferred to this function. */
1024 schedule_packet_in(struct ofconn *ofconn, const struct dpif_upcall *upcall,
1025 const struct flow *flow, struct ofpbuf *rw_packet)
1027 struct connmgr *mgr = ofconn->connmgr;
1028 struct ofputil_packet_in pin;
1030 /* Figure out the easy parts. */
1031 pin.packet = upcall->packet;
1032 pin.in_port = odp_port_to_ofp_port(flow->in_port);
1033 pin.reason = upcall->type == DPIF_UC_MISS ? OFPR_NO_MATCH : OFPR_ACTION;
1035 /* Get OpenFlow buffer_id. */
1036 if (upcall->type == DPIF_UC_ACTION) {
1037 pin.buffer_id = UINT32_MAX;
1038 } else if (mgr->fail_open && fail_open_is_active(mgr->fail_open)) {
1039 pin.buffer_id = pktbuf_get_null();
1040 } else if (!ofconn->pktbuf) {
1041 pin.buffer_id = UINT32_MAX;
1043 pin.buffer_id = pktbuf_save(ofconn->pktbuf, upcall->packet,
1047 /* Figure out how much of the packet to send. */
1048 pin.send_len = upcall->packet->size;
1049 if (pin.buffer_id != UINT32_MAX) {
1050 pin.send_len = MIN(pin.send_len, ofconn->miss_send_len);
1052 if (upcall->type == DPIF_UC_ACTION) {
1053 pin.send_len = MIN(pin.send_len, upcall->userdata);
1056 /* Make OFPT_PACKET_IN and hand over to packet scheduler. It might
1057 * immediately call into do_send_packet_in() or it might buffer it for a
1058 * while (until a later call to pinsched_run()). */
1059 pinsched_send(ofconn->schedulers[upcall->type == DPIF_UC_MISS ? 0 : 1],
1060 flow->in_port, ofputil_encode_packet_in(&pin, rw_packet),
1061 do_send_packet_in, ofconn);
1064 /* Fail-open settings. */
1066 /* Returns the failure handling mode (OFPROTO_FAIL_SECURE or
1067 * OFPROTO_FAIL_STANDALONE) for 'mgr'. */
1068 enum ofproto_fail_mode
1069 connmgr_get_fail_mode(const struct connmgr *mgr)
1071 return mgr->fail_mode;
1074 /* Sets the failure handling mode for 'mgr' to 'fail_mode' (either
1075 * OFPROTO_FAIL_SECURE or OFPROTO_FAIL_STANDALONE). */
1077 connmgr_set_fail_mode(struct connmgr *mgr, enum ofproto_fail_mode fail_mode)
1079 mgr->fail_mode = fail_mode;
1080 update_fail_open(mgr);
1083 /* Fail-open implementation. */
1085 /* Returns the longest probe interval among the primary controllers configured
1086 * on 'mgr'. Returns 0 if there are no primary controllers. */
1088 connmgr_get_max_probe_interval(const struct connmgr *mgr)
1090 const struct ofconn *ofconn;
1091 int max_probe_interval;
1093 max_probe_interval = 0;
1094 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1095 int probe_interval = rconn_get_probe_interval(ofconn->rconn);
1096 max_probe_interval = MAX(max_probe_interval, probe_interval);
1098 return max_probe_interval;
1101 /* Returns the number of seconds for which all of 'mgr's primary controllers
1102 * have been disconnected. Returns 0 if 'mgr' has no primary controllers. */
1104 connmgr_failure_duration(const struct connmgr *mgr)
1106 const struct ofconn *ofconn;
1107 int min_failure_duration;
1109 if (!connmgr_has_controllers(mgr)) {
1113 min_failure_duration = INT_MAX;
1114 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1115 int failure_duration = rconn_failure_duration(ofconn->rconn);
1116 min_failure_duration = MIN(min_failure_duration, failure_duration);
1118 return min_failure_duration;
1121 /* Returns true if at least one primary controller is connected (regardless of
1122 * whether those controllers are believed to have authenticated and accepted
1123 * this switch), false if none of them are connected. */
1125 connmgr_is_any_controller_connected(const struct connmgr *mgr)
1127 const struct ofconn *ofconn;
1129 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1130 if (rconn_is_connected(ofconn->rconn)) {
1137 /* Returns true if at least one primary controller is believed to have
1138 * authenticated and accepted this switch, false otherwise. */
1140 connmgr_is_any_controller_admitted(const struct connmgr *mgr)
1142 const struct ofconn *ofconn;
1144 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1145 if (rconn_is_admitted(ofconn->rconn)) {
1152 /* Sends 'packet' to each controller connected to 'mgr'. Takes ownership of
1155 connmgr_broadcast(struct connmgr *mgr, struct ofpbuf *packet)
1157 struct ofconn *ofconn, *prev;
1160 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
1162 ofconn_send_reply(ofconn, ofpbuf_clone(packet));
1164 if (rconn_is_connected(ofconn->rconn)) {
1169 ofconn_send_reply(prev, packet);
1171 ofpbuf_delete(packet);
1175 /* In-band configuration. */
1177 static bool any_extras_changed(const struct connmgr *,
1178 const struct sockaddr_in *extras, size_t n);
1180 /* Sets the 'n' TCP port addresses in 'extras' as ones to which 'mgr''s
1181 * in-band control should guarantee access, in the same way that in-band
1182 * control guarantees access to OpenFlow controllers. */
1184 connmgr_set_extra_in_band_remotes(struct connmgr *mgr,
1185 const struct sockaddr_in *extras, size_t n)
1187 if (!any_extras_changed(mgr, extras, n)) {
1191 free(mgr->extra_in_band_remotes);
1192 mgr->n_extra_remotes = n;
1193 mgr->extra_in_band_remotes = xmemdup(extras, n * sizeof *extras);
1195 update_in_band_remotes(mgr);
1198 /* Sets the OpenFlow queue used by flows set up by in-band control on
1199 * 'mgr' to 'queue_id'. If 'queue_id' is negative, then in-band control
1200 * flows will use the default queue. */
1202 connmgr_set_in_band_queue(struct connmgr *mgr, int queue_id)
1204 if (queue_id != mgr->in_band_queue) {
1205 mgr->in_band_queue = queue_id;
1206 update_in_band_remotes(mgr);
1211 any_extras_changed(const struct connmgr *mgr,
1212 const struct sockaddr_in *extras, size_t n)
1216 if (n != mgr->n_extra_remotes) {
1220 for (i = 0; i < n; i++) {
1221 const struct sockaddr_in *old = &mgr->extra_in_band_remotes[i];
1222 const struct sockaddr_in *new = &extras[i];
1224 if (old->sin_addr.s_addr != new->sin_addr.s_addr ||
1225 old->sin_port != new->sin_port) {
1233 /* In-band implementation. */
1236 connmgr_msg_in_hook(struct connmgr *mgr, const struct flow *flow,
1237 const struct ofpbuf *packet)
1239 return mgr->in_band && in_band_msg_in_hook(mgr->in_band, flow, packet);
1243 connmgr_may_set_up_flow(struct connmgr *mgr, const struct flow *flow,
1244 const struct nlattr *odp_actions,
1247 return !mgr->in_band || in_band_rule_check(flow, odp_actions, actions_len);
1250 /* Fail-open and in-band implementation. */
1252 /* Called by 'ofproto' after all flows have been flushed, to allow fail-open
1253 * and in-band control to re-create their flows. */
1255 connmgr_flushed(struct connmgr *mgr)
1258 in_band_flushed(mgr->in_band);
1260 if (mgr->fail_open) {
1261 fail_open_flushed(mgr->fail_open);
1265 /* Creates a new ofservice for 'target' in 'mgr'. Returns 0 if successful,
1266 * otherwise a positive errno value.
1268 * ofservice_reconfigure() must be called to fully configure the new
1271 ofservice_create(struct connmgr *mgr, const char *target)
1273 struct ofservice *ofservice;
1274 struct pvconn *pvconn;
1277 error = pvconn_open(target, &pvconn);
1282 ofservice = xzalloc(sizeof *ofservice);
1283 hmap_insert(&mgr->services, &ofservice->node, hash_string(target, 0));
1284 ofservice->pvconn = pvconn;
1290 ofservice_destroy(struct connmgr *mgr, struct ofservice *ofservice)
1292 hmap_remove(&mgr->services, &ofservice->node);
1293 pvconn_close(ofservice->pvconn);
1298 ofservice_reconfigure(struct ofservice *ofservice,
1299 const struct ofproto_controller *c)
1301 ofservice->probe_interval = c->probe_interval;
1302 ofservice->rate_limit = c->rate_limit;
1303 ofservice->burst_limit = c->burst_limit;
1306 /* Finds and returns the ofservice within 'mgr' that has the given
1307 * 'target', or a null pointer if none exists. */
1308 static struct ofservice *
1309 ofservice_lookup(struct connmgr *mgr, const char *target)
1311 struct ofservice *ofservice;
1313 HMAP_FOR_EACH_WITH_HASH (ofservice, node, hash_string(target, 0),
1315 if (!strcmp(pvconn_get_name(ofservice->pvconn), target)) {