2 * Copyright (c) 2009, 2010, 2011 Nicira Networks.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
26 #include "fail-open.h"
32 #include "poll-loop.h"
40 VLOG_DEFINE_THIS_MODULE(connmgr);
41 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
43 COVERAGE_DEFINE(ofconn_stuck);
45 /* An OpenFlow connection. */
47 struct connmgr *connmgr; /* Connection's manager. */
48 struct list node; /* In struct connmgr's "all_conns" list. */
49 struct rconn *rconn; /* OpenFlow connection. */
50 enum ofconn_type type; /* Type. */
51 enum nx_flow_format flow_format; /* Currently selected flow format. */
53 /* OFPT_PACKET_IN related data. */
54 struct rconn_packet_counter *packet_in_counter; /* # queued on 'rconn'. */
55 #define N_SCHEDULERS 2
56 struct pinsched *schedulers[N_SCHEDULERS];
57 struct pktbuf *pktbuf; /* OpenFlow packet buffers. */
58 int miss_send_len; /* Bytes to send of buffered packets. */
60 /* Number of OpenFlow messages queued on 'rconn' as replies to OpenFlow
61 * requests, and the maximum number before we stop reading OpenFlow
63 #define OFCONN_REPLY_MAX 100
64 struct rconn_packet_counter *reply_counter;
66 /* type == OFCONN_PRIMARY only. */
67 enum nx_role role; /* Role. */
68 struct hmap_node hmap_node; /* In struct connmgr's "controllers" map. */
69 enum ofproto_band band; /* In-band or out-of-band? */
72 static struct ofconn *ofconn_create(struct connmgr *, struct rconn *,
74 static void ofconn_destroy(struct ofconn *);
76 static void ofconn_reconfigure(struct ofconn *,
77 const struct ofproto_controller *);
79 static void ofconn_run(struct ofconn *,
80 void (*handle_openflow)(struct ofconn *,
81 struct ofpbuf *ofp_msg));
82 static void ofconn_wait(struct ofconn *);
84 static const char *ofconn_get_target(const struct ofconn *);
85 static char *ofconn_make_name(const struct connmgr *, const char *target);
87 static void ofconn_set_rate_limit(struct ofconn *, int rate, int burst);
89 static bool ofconn_receives_async_msgs(const struct ofconn *);
91 static void ofconn_send(const struct ofconn *, struct ofpbuf *,
92 struct rconn_packet_counter *);
94 static void do_send_packet_in(struct ofpbuf *, void *ofconn_);
96 /* A listener for incoming OpenFlow "service" connections. */
98 struct hmap_node node; /* In struct connmgr's "services" hmap. */
99 struct pvconn *pvconn; /* OpenFlow connection listener. */
101 /* These are not used by ofservice directly. They are settings for
102 * accepted "struct ofconn"s from the pvconn. */
103 int probe_interval; /* Max idle time before probing, in seconds. */
104 int rate_limit; /* Max packet-in rate in packets per second. */
105 int burst_limit; /* Limit on accumulating packet credits. */
108 static void ofservice_reconfigure(struct ofservice *,
109 const struct ofproto_controller *);
110 static int ofservice_create(struct connmgr *, const char *target);
111 static void ofservice_destroy(struct connmgr *, struct ofservice *);
112 static struct ofservice *ofservice_lookup(struct connmgr *,
115 /* Connection manager for an OpenFlow switch. */
117 struct ofproto *ofproto;
119 char *local_port_name;
121 /* OpenFlow connections. */
122 struct hmap controllers; /* Controller "struct ofconn"s. */
123 struct list all_conns; /* Contains "struct ofconn"s. */
125 /* OpenFlow listeners. */
126 struct hmap services; /* Contains "struct ofservice"s. */
127 struct pvconn **snoops;
131 struct fail_open *fail_open;
132 enum ofproto_fail_mode fail_mode;
134 /* In-band control. */
135 struct in_band *in_band;
136 long long int next_in_band_update;
137 struct sockaddr_in *extra_in_band_remotes;
138 size_t n_extra_remotes;
142 static void update_in_band_remotes(struct connmgr *);
143 static void add_snooper(struct connmgr *, struct vconn *);
145 /* Creates and returns a new connection manager owned by 'ofproto'. 'name' is
146 * a name for the ofproto suitable for using in log messages.
147 * 'local_port_name' is the name of the local port (OFPP_LOCAL) within
150 connmgr_create(struct ofproto *ofproto,
151 const char *name, const char *local_port_name)
155 mgr = xmalloc(sizeof *mgr);
156 mgr->ofproto = ofproto;
157 mgr->name = xstrdup(name);
158 mgr->local_port_name = xstrdup(local_port_name);
160 hmap_init(&mgr->controllers);
161 list_init(&mgr->all_conns);
163 hmap_init(&mgr->services);
167 mgr->fail_open = NULL;
168 mgr->fail_mode = OFPROTO_FAIL_SECURE;
171 mgr->next_in_band_update = LLONG_MAX;
172 mgr->extra_in_band_remotes = NULL;
173 mgr->n_extra_remotes = 0;
174 mgr->in_band_queue = -1;
179 /* Frees 'mgr' and all of its resources. */
181 connmgr_destroy(struct connmgr *mgr)
183 struct ofservice *ofservice, *next_ofservice;
184 struct ofconn *ofconn, *next_ofconn;
191 LIST_FOR_EACH_SAFE (ofconn, next_ofconn, node, &mgr->all_conns) {
192 ofconn_destroy(ofconn);
194 hmap_destroy(&mgr->controllers);
196 HMAP_FOR_EACH_SAFE (ofservice, next_ofservice, node, &mgr->services) {
197 ofservice_destroy(mgr, ofservice);
199 hmap_destroy(&mgr->services);
201 for (i = 0; i < mgr->n_snoops; i++) {
202 pvconn_close(mgr->snoops[i]);
206 fail_open_destroy(mgr->fail_open);
207 mgr->fail_open = NULL;
209 in_band_destroy(mgr->in_band);
211 free(mgr->extra_in_band_remotes);
213 free(mgr->local_port_name);
218 /* Does all of the periodic maintenance required by 'mgr'. Calls
219 * 'handle_openflow' for each message received on an OpenFlow connection,
220 * passing along the OpenFlow connection itself and the message that was sent.
221 * The 'handle_openflow' callback must not free the message. */
223 connmgr_run(struct connmgr *mgr,
224 void (*handle_openflow)(struct ofconn *, struct ofpbuf *ofp_msg))
226 struct ofconn *ofconn, *next_ofconn;
227 struct ofservice *ofservice;
231 if (time_msec() >= mgr->next_in_band_update) {
232 update_in_band_remotes(mgr);
234 in_band_run(mgr->in_band);
237 LIST_FOR_EACH_SAFE (ofconn, next_ofconn, node, &mgr->all_conns) {
238 ofconn_run(ofconn, handle_openflow);
241 /* Fail-open maintenance. Do this after processing the ofconns since
242 * fail-open checks the status of the controller rconn. */
243 if (mgr->fail_open) {
244 fail_open_run(mgr->fail_open);
247 HMAP_FOR_EACH (ofservice, node, &mgr->services) {
251 retval = pvconn_accept(ofservice->pvconn, OFP_VERSION, &vconn);
256 rconn = rconn_create(ofservice->probe_interval, 0);
257 name = ofconn_make_name(mgr, vconn_get_name(vconn));
258 rconn_connect_unreliably(rconn, vconn, name);
261 ofconn = ofconn_create(mgr, rconn, OFCONN_SERVICE);
262 ofconn_set_rate_limit(ofconn, ofservice->rate_limit,
263 ofservice->burst_limit);
264 } else if (retval != EAGAIN) {
265 VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
269 for (i = 0; i < mgr->n_snoops; i++) {
273 retval = pvconn_accept(mgr->snoops[i], OFP_VERSION, &vconn);
275 add_snooper(mgr, vconn);
276 } else if (retval != EAGAIN) {
277 VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
282 /* Causes the poll loop to wake up when connmgr_run() needs to run. */
284 connmgr_wait(struct connmgr *mgr)
286 struct ofservice *ofservice;
287 struct ofconn *ofconn;
290 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
294 poll_timer_wait_until(mgr->next_in_band_update);
295 in_band_wait(mgr->in_band);
297 if (mgr->fail_open) {
298 fail_open_wait(mgr->fail_open);
300 HMAP_FOR_EACH (ofservice, node, &mgr->services) {
301 pvconn_wait(ofservice->pvconn);
303 for (i = 0; i < mgr->n_snoops; i++) {
304 pvconn_wait(mgr->snoops[i]);
308 /* Returns the ofproto that owns 'ofconn''s connmgr. */
310 ofconn_get_ofproto(const struct ofconn *ofconn)
312 return ofconn->connmgr->ofproto;
315 /* OpenFlow configuration. */
317 static void add_controller(struct connmgr *, const char *target);
318 static struct ofconn *find_controller_by_target(struct connmgr *,
320 static void update_fail_open(struct connmgr *);
321 static int set_pvconns(struct pvconn ***pvconnsp, size_t *n_pvconnsp,
322 const struct sset *);
324 /* Returns true if 'mgr' has any configured primary controllers.
326 * Service controllers do not count, but configured primary controllers do
327 * count whether or not they are currently connected. */
329 connmgr_has_controllers(const struct connmgr *mgr)
331 return !hmap_is_empty(&mgr->controllers);
334 /* Initializes 'info' and populates it with information about each configured
335 * primary controller. The keys in 'info' are the controllers' targets; the
336 * data values are corresponding "struct ofproto_controller_info".
338 * The caller owns 'info' and everything in it and should free it when it is no
341 connmgr_get_controller_info(struct connmgr *mgr, struct shash *info)
343 const struct ofconn *ofconn;
347 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
348 const struct rconn *rconn = ofconn->rconn;
349 time_t now = time_now();
350 time_t last_connection = rconn_get_last_connection(rconn);
351 time_t last_disconnect = rconn_get_last_disconnect(rconn);
352 int last_error = rconn_get_last_error(rconn);
353 struct ofproto_controller_info *cinfo = xmalloc(sizeof *cinfo);
355 shash_add(info, rconn_get_target(rconn), cinfo);
357 cinfo->is_connected = rconn_is_connected(rconn);
358 cinfo->role = ofconn->role;
363 cinfo->pairs.keys[cinfo->pairs.n] = "last_error";
364 cinfo->pairs.values[cinfo->pairs.n++] =
365 xstrdup(ovs_retval_to_string(last_error));
368 cinfo->pairs.keys[cinfo->pairs.n] = "state";
369 cinfo->pairs.values[cinfo->pairs.n++] =
370 xstrdup(rconn_get_state(rconn));
372 if (last_connection != TIME_MIN) {
373 cinfo->pairs.keys[cinfo->pairs.n] = "sec_since_connect";
374 cinfo->pairs.values[cinfo->pairs.n++]
375 = xasprintf("%ld", (long int) (now - last_connection));
378 if (last_disconnect != TIME_MIN) {
379 cinfo->pairs.keys[cinfo->pairs.n] = "sec_since_disconnect";
380 cinfo->pairs.values[cinfo->pairs.n++]
381 = xasprintf("%ld", (long int) (now - last_disconnect));
386 /* Changes 'mgr''s set of controllers to the 'n_controllers' controllers in
389 connmgr_set_controllers(struct connmgr *mgr,
390 const struct ofproto_controller *controllers,
391 size_t n_controllers)
393 struct shash new_controllers;
394 struct ofconn *ofconn, *next_ofconn;
395 struct ofservice *ofservice, *next_ofservice;
399 /* Create newly configured controllers and services.
400 * Create a name to ofproto_controller mapping in 'new_controllers'. */
401 shash_init(&new_controllers);
402 for (i = 0; i < n_controllers; i++) {
403 const struct ofproto_controller *c = &controllers[i];
405 if (!vconn_verify_name(c->target)) {
406 if (!find_controller_by_target(mgr, c->target)) {
407 add_controller(mgr, c->target);
409 } else if (!pvconn_verify_name(c->target)) {
410 if (!ofservice_lookup(mgr, c->target)) {
411 ofservice_create(mgr, c->target);
414 VLOG_WARN_RL(&rl, "%s: unsupported controller \"%s\"",
415 mgr->name, c->target);
419 shash_add_once(&new_controllers, c->target, &controllers[i]);
422 /* Delete controllers that are no longer configured.
423 * Update configuration of all now-existing controllers. */
425 HMAP_FOR_EACH_SAFE (ofconn, next_ofconn, hmap_node, &mgr->controllers) {
426 struct ofproto_controller *c;
428 c = shash_find_data(&new_controllers, ofconn_get_target(ofconn));
430 ofconn_destroy(ofconn);
432 ofconn_reconfigure(ofconn, c);
436 /* Delete services that are no longer configured.
437 * Update configuration of all now-existing services. */
438 HMAP_FOR_EACH_SAFE (ofservice, next_ofservice, node, &mgr->services) {
439 struct ofproto_controller *c;
441 c = shash_find_data(&new_controllers,
442 pvconn_get_name(ofservice->pvconn));
444 ofservice_destroy(mgr, ofservice);
446 ofservice_reconfigure(ofservice, c);
450 shash_destroy(&new_controllers);
452 update_in_band_remotes(mgr);
453 update_fail_open(mgr);
456 /* Drops the connections between 'mgr' and all of its primary and secondary
457 * controllers, forcing them to reconnect. */
459 connmgr_reconnect(const struct connmgr *mgr)
461 struct ofconn *ofconn;
463 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
464 rconn_reconnect(ofconn->rconn);
468 /* Sets the "snoops" for 'mgr' to the pvconn targets listed in 'snoops'.
470 * A "snoop" is a pvconn to which every OpenFlow message to or from the most
471 * important controller on 'mgr' is mirrored. */
473 connmgr_set_snoops(struct connmgr *mgr, const struct sset *snoops)
475 return set_pvconns(&mgr->snoops, &mgr->n_snoops, snoops);
478 /* Adds each of the snoops currently configured on 'mgr' to 'snoops'. */
480 connmgr_get_snoops(const struct connmgr *mgr, struct sset *snoops)
484 for (i = 0; i < mgr->n_snoops; i++) {
485 sset_add(snoops, pvconn_get_name(mgr->snoops[i]));
489 /* Returns true if 'mgr' has at least one snoop, false if it has none. */
491 connmgr_has_snoops(const struct connmgr *mgr)
493 return mgr->n_snoops > 0;
496 /* Creates a new controller for 'target' in 'mgr'. update_controller() needs
497 * to be called later to finish the new ofconn's configuration. */
499 add_controller(struct connmgr *mgr, const char *target)
501 char *name = ofconn_make_name(mgr, target);
502 struct ofconn *ofconn;
504 ofconn = ofconn_create(mgr, rconn_create(5, 8), OFCONN_PRIMARY);
505 ofconn->pktbuf = pktbuf_create();
506 ofconn->miss_send_len = OFP_DEFAULT_MISS_SEND_LEN;
507 rconn_connect(ofconn->rconn, target, name);
508 hmap_insert(&mgr->controllers, &ofconn->hmap_node, hash_string(target, 0));
513 static struct ofconn *
514 find_controller_by_target(struct connmgr *mgr, const char *target)
516 struct ofconn *ofconn;
518 HMAP_FOR_EACH_WITH_HASH (ofconn, hmap_node,
519 hash_string(target, 0), &mgr->controllers) {
520 if (!strcmp(ofconn_get_target(ofconn), target)) {
528 update_in_band_remotes(struct connmgr *mgr)
530 struct sockaddr_in *addrs;
531 size_t max_addrs, n_addrs;
532 struct ofconn *ofconn;
535 /* Allocate enough memory for as many remotes as we could possibly have. */
536 max_addrs = mgr->n_extra_remotes + hmap_count(&mgr->controllers);
537 addrs = xmalloc(max_addrs * sizeof *addrs);
540 /* Add all the remotes. */
541 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
542 struct sockaddr_in *sin = &addrs[n_addrs];
544 if (ofconn->band == OFPROTO_OUT_OF_BAND) {
548 sin->sin_addr.s_addr = rconn_get_remote_ip(ofconn->rconn);
549 if (sin->sin_addr.s_addr) {
550 sin->sin_port = rconn_get_remote_port(ofconn->rconn);
554 for (i = 0; i < mgr->n_extra_remotes; i++) {
555 addrs[n_addrs++] = mgr->extra_in_band_remotes[i];
558 /* Create or update or destroy in-band. */
561 in_band_create(mgr->ofproto, mgr->local_port_name, &mgr->in_band);
564 in_band_set_remotes(mgr->in_band, addrs, n_addrs);
566 in_band_set_queue(mgr->in_band, mgr->in_band_queue);
567 mgr->next_in_band_update = time_msec() + 1000;
569 in_band_destroy(mgr->in_band);
578 update_fail_open(struct connmgr *mgr)
580 if (connmgr_has_controllers(mgr)
581 && mgr->fail_mode == OFPROTO_FAIL_STANDALONE) {
582 if (!mgr->fail_open) {
583 mgr->fail_open = fail_open_create(mgr->ofproto, mgr);
586 fail_open_destroy(mgr->fail_open);
587 mgr->fail_open = NULL;
592 set_pvconns(struct pvconn ***pvconnsp, size_t *n_pvconnsp,
593 const struct sset *sset)
595 struct pvconn **pvconns = *pvconnsp;
596 size_t n_pvconns = *n_pvconnsp;
601 for (i = 0; i < n_pvconns; i++) {
602 pvconn_close(pvconns[i]);
606 pvconns = xmalloc(sset_count(sset) * sizeof *pvconns);
608 SSET_FOR_EACH (name, sset) {
609 struct pvconn *pvconn;
612 error = pvconn_open(name, &pvconn);
614 pvconns[n_pvconns++] = pvconn;
616 VLOG_ERR("failed to listen on %s: %s", name, strerror(error));
624 *n_pvconnsp = n_pvconns;
629 /* Returns a "preference level" for snooping 'ofconn'. A higher return value
630 * means that 'ofconn' is more interesting for monitoring than a lower return
633 snoop_preference(const struct ofconn *ofconn)
635 switch (ofconn->role) {
643 /* Shouldn't happen. */
648 /* One of 'mgr''s "snoop" pvconns has accepted a new connection on 'vconn'.
649 * Connects this vconn to a controller. */
651 add_snooper(struct connmgr *mgr, struct vconn *vconn)
653 struct ofconn *ofconn, *best;
655 /* Pick a controller for monitoring. */
657 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
658 if (ofconn->type == OFCONN_PRIMARY
659 && (!best || snoop_preference(ofconn) > snoop_preference(best))) {
665 rconn_add_monitor(best->rconn, vconn);
667 VLOG_INFO_RL(&rl, "no controller connection to snoop");
672 /* Public ofconn functions. */
674 /* Returns the connection type, either OFCONN_PRIMARY or OFCONN_SERVICE. */
676 ofconn_get_type(const struct ofconn *ofconn)
681 /* Returns the role configured for 'ofconn'.
683 * The default role, if no other role has been set, is NX_ROLE_OTHER. */
685 ofconn_get_role(const struct ofconn *ofconn)
690 /* Changes 'ofconn''s role to 'role'. If 'role' is NX_ROLE_MASTER then any
691 * existing master is demoted to a slave. */
693 ofconn_set_role(struct ofconn *ofconn, enum nx_role role)
695 if (role == NX_ROLE_MASTER) {
696 struct ofconn *other;
698 HMAP_FOR_EACH (other, hmap_node, &ofconn->connmgr->controllers) {
699 if (other->role == NX_ROLE_MASTER) {
700 other->role = NX_ROLE_SLAVE;
707 /* Returns the currently configured flow format for 'ofconn', one of NXFF_*.
709 * The default, if no other format has been set, is NXFF_OPENFLOW10. */
711 ofconn_get_flow_format(struct ofconn *ofconn)
713 return ofconn->flow_format;
716 /* Sets the flow format for 'ofconn' to 'flow_format' (one of NXFF_*). */
718 ofconn_set_flow_format(struct ofconn *ofconn, enum nx_flow_format flow_format)
720 ofconn->flow_format = flow_format;
723 /* Returns the default miss send length for 'ofconn'. */
725 ofconn_get_miss_send_len(const struct ofconn *ofconn)
727 return ofconn->miss_send_len;
730 /* Sets the default miss send length for 'ofconn' to 'miss_send_len'. */
732 ofconn_set_miss_send_len(struct ofconn *ofconn, int miss_send_len)
734 ofconn->miss_send_len = miss_send_len;
737 /* Sends 'msg' on 'ofconn', accounting it as a reply. (If there is a
738 * sufficient number of OpenFlow replies in-flight on a single ofconn, then the
739 * connmgr will stop accepting new OpenFlow requests on that ofconn until the
740 * controller has accepted some of the replies.) */
742 ofconn_send_reply(const struct ofconn *ofconn, struct ofpbuf *msg)
744 ofconn_send(ofconn, msg, ofconn->reply_counter);
747 /* Same as pktbuf_retrieve(), using the pktbuf owned by 'ofconn'. */
749 ofconn_pktbuf_retrieve(struct ofconn *ofconn, uint32_t id,
750 struct ofpbuf **bufferp, uint16_t *in_port)
752 return pktbuf_retrieve(ofconn->pktbuf, id, bufferp, in_port);
755 /* Private ofconn functions. */
758 ofconn_get_target(const struct ofconn *ofconn)
760 return rconn_get_target(ofconn->rconn);
763 static struct ofconn *
764 ofconn_create(struct connmgr *mgr, struct rconn *rconn, enum ofconn_type type)
766 struct ofconn *ofconn = xzalloc(sizeof *ofconn);
767 ofconn->connmgr = mgr;
768 list_push_back(&mgr->all_conns, &ofconn->node);
769 ofconn->rconn = rconn;
771 ofconn->flow_format = NXFF_OPENFLOW10;
772 ofconn->role = NX_ROLE_OTHER;
773 ofconn->packet_in_counter = rconn_packet_counter_create ();
774 ofconn->pktbuf = NULL;
775 ofconn->miss_send_len = 0;
776 ofconn->reply_counter = rconn_packet_counter_create ();
781 ofconn_destroy(struct ofconn *ofconn)
783 if (ofconn->type == OFCONN_PRIMARY) {
784 hmap_remove(&ofconn->connmgr->controllers, &ofconn->hmap_node);
787 list_remove(&ofconn->node);
788 rconn_destroy(ofconn->rconn);
789 rconn_packet_counter_destroy(ofconn->packet_in_counter);
790 rconn_packet_counter_destroy(ofconn->reply_counter);
791 pktbuf_destroy(ofconn->pktbuf);
795 /* Reconfigures 'ofconn' to match 'c'. 'ofconn' and 'c' must have the same
798 ofconn_reconfigure(struct ofconn *ofconn, const struct ofproto_controller *c)
802 ofconn->band = c->band;
804 rconn_set_max_backoff(ofconn->rconn, c->max_backoff);
806 probe_interval = c->probe_interval ? MAX(c->probe_interval, 5) : 0;
807 rconn_set_probe_interval(ofconn->rconn, probe_interval);
809 ofconn_set_rate_limit(ofconn, c->rate_limit, c->burst_limit);
813 ofconn_run(struct ofconn *ofconn,
814 void (*handle_openflow)(struct ofconn *, struct ofpbuf *ofp_msg))
816 struct connmgr *mgr = ofconn->connmgr;
820 for (i = 0; i < N_SCHEDULERS; i++) {
821 pinsched_run(ofconn->schedulers[i], do_send_packet_in, ofconn);
824 rconn_run(ofconn->rconn);
826 if (rconn_packet_counter_read (ofconn->reply_counter) < OFCONN_REPLY_MAX) {
827 /* Limit the number of iterations to prevent other tasks from
829 for (iteration = 0; iteration < 50; iteration++) {
830 struct ofpbuf *of_msg = rconn_recv(ofconn->rconn);
834 if (mgr->fail_open) {
835 fail_open_maybe_recover(mgr->fail_open);
837 handle_openflow(ofconn, of_msg);
838 ofpbuf_delete(of_msg);
842 if (!rconn_is_alive(ofconn->rconn)) {
843 ofconn_destroy(ofconn);
848 ofconn_wait(struct ofconn *ofconn)
852 for (i = 0; i < N_SCHEDULERS; i++) {
853 pinsched_wait(ofconn->schedulers[i]);
855 rconn_run_wait(ofconn->rconn);
856 if (rconn_packet_counter_read (ofconn->reply_counter) < OFCONN_REPLY_MAX) {
857 rconn_recv_wait(ofconn->rconn);
859 COVERAGE_INC(ofconn_stuck);
863 /* Returns true if 'ofconn' should receive asynchronous messages. */
865 ofconn_receives_async_msgs(const struct ofconn *ofconn)
867 if (!rconn_is_connected(ofconn->rconn)) {
869 } else if (ofconn->type == OFCONN_PRIMARY) {
870 /* Primary controllers always get asynchronous messages unless they
871 * have configured themselves as "slaves". */
872 return ofconn->role != NX_ROLE_SLAVE;
874 /* Service connections don't get asynchronous messages unless they have
875 * explicitly asked for them by setting a nonzero miss send length. */
876 return ofconn->miss_send_len > 0;
880 /* Returns a human-readable name for an OpenFlow connection between 'mgr' and
881 * 'target', suitable for use in log messages for identifying the connection.
883 * The name is dynamically allocated. The caller should free it (with free())
884 * when it is no longer needed. */
886 ofconn_make_name(const struct connmgr *mgr, const char *target)
888 return xasprintf("%s<->%s", mgr->name, target);
892 ofconn_set_rate_limit(struct ofconn *ofconn, int rate, int burst)
896 for (i = 0; i < N_SCHEDULERS; i++) {
897 struct pinsched **s = &ofconn->schedulers[i];
901 *s = pinsched_create(rate, burst);
903 pinsched_set_limits(*s, rate, burst);
906 pinsched_destroy(*s);
913 ofconn_send(const struct ofconn *ofconn, struct ofpbuf *msg,
914 struct rconn_packet_counter *counter)
916 update_openflow_length(msg);
917 if (rconn_send(ofconn->rconn, msg, counter)) {
922 /* Sending asynchronous messages. */
924 static void schedule_packet_in(struct ofconn *, const struct dpif_upcall *,
925 const struct flow *, struct ofpbuf *rw_packet);
927 /* Sends an OFPT_PORT_STATUS message with 'opp' and 'reason' to appropriate
928 * controllers managed by 'mgr'.
930 * 'opp' is in *HOST* byte order. */
932 connmgr_send_port_status(struct connmgr *mgr, const struct ofp_phy_port *opp,
935 /* XXX Should limit the number of queued port status change messages. */
936 struct ofconn *ofconn;
938 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
939 struct ofp_port_status *ops;
942 /* Primary controllers, even slaves, should always get port status
943 updates. Otherwise obey ofconn_receives_async_msgs(). */
944 if (ofconn->type != OFCONN_PRIMARY
945 && !ofconn_receives_async_msgs(ofconn)) {
949 ops = make_openflow_xid(sizeof *ops, OFPT_PORT_STATUS, 0, &b);
950 ops->reason = reason;
952 hton_ofp_phy_port(&ops->desc);
953 ofconn_send(ofconn, b, NULL);
957 /* Sends an OFPT_FLOW_REMOVED or NXT_FLOW_REMOVED message based on 'fr' to
958 * appropriate controllers managed by 'mgr'. */
960 connmgr_send_flow_removed(struct connmgr *mgr,
961 const struct ofputil_flow_removed *fr)
963 struct ofconn *ofconn;
965 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
968 if (!ofconn_receives_async_msgs(ofconn)) {
972 /* Account flow expirations as replies to OpenFlow requests. That
973 * works because preventing OpenFlow requests from being processed also
974 * prevents new flows from being added (and expiring). (It also
975 * prevents processing OpenFlow requests that would not add new flows,
976 * so it is imperfect.) */
977 msg = ofputil_encode_flow_removed(fr, ofconn->flow_format);
978 ofconn_send_reply(ofconn, msg);
982 /* Given 'upcall', of type DPIF_UC_ACTION or DPIF_UC_MISS, sends an
983 * OFPT_PACKET_IN message to each OpenFlow controller as necessary according to
984 * their individual configurations.
986 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
987 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
988 * of 'rw_packet' is transferred to this function. */
990 connmgr_send_packet_in(struct connmgr *mgr, const struct dpif_upcall *upcall,
991 const struct flow *flow, struct ofpbuf *rw_packet)
993 struct ofconn *ofconn, *prev;
996 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
997 if (ofconn_receives_async_msgs(ofconn)) {
999 schedule_packet_in(prev, upcall, flow, NULL);
1005 schedule_packet_in(prev, upcall, flow, rw_packet);
1007 ofpbuf_delete(rw_packet);
1011 /* pinsched callback for sending 'ofp_packet_in' on 'ofconn'. */
1013 do_send_packet_in(struct ofpbuf *ofp_packet_in, void *ofconn_)
1015 struct ofconn *ofconn = ofconn_;
1017 rconn_send_with_limit(ofconn->rconn, ofp_packet_in,
1018 ofconn->packet_in_counter, 100);
1021 /* Takes 'upcall', whose packet has the flow specified by 'flow', composes an
1022 * OpenFlow packet-in message from it, and passes it to 'ofconn''s packet
1023 * scheduler for sending.
1025 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
1026 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
1027 * of 'rw_packet' is transferred to this function. */
1029 schedule_packet_in(struct ofconn *ofconn, const struct dpif_upcall *upcall,
1030 const struct flow *flow, struct ofpbuf *rw_packet)
1032 struct connmgr *mgr = ofconn->connmgr;
1033 struct ofputil_packet_in pin;
1035 /* Figure out the easy parts. */
1036 pin.packet = upcall->packet;
1037 pin.in_port = odp_port_to_ofp_port(flow->in_port);
1038 pin.reason = upcall->type == DPIF_UC_MISS ? OFPR_NO_MATCH : OFPR_ACTION;
1040 /* Get OpenFlow buffer_id. */
1041 if (upcall->type == DPIF_UC_ACTION) {
1042 pin.buffer_id = UINT32_MAX;
1043 } else if (mgr->fail_open && fail_open_is_active(mgr->fail_open)) {
1044 pin.buffer_id = pktbuf_get_null();
1045 } else if (!ofconn->pktbuf) {
1046 pin.buffer_id = UINT32_MAX;
1048 pin.buffer_id = pktbuf_save(ofconn->pktbuf, upcall->packet,
1052 /* Figure out how much of the packet to send. */
1053 pin.send_len = upcall->packet->size;
1054 if (pin.buffer_id != UINT32_MAX) {
1055 pin.send_len = MIN(pin.send_len, ofconn->miss_send_len);
1057 if (upcall->type == DPIF_UC_ACTION) {
1058 pin.send_len = MIN(pin.send_len, upcall->userdata);
1061 /* Make OFPT_PACKET_IN and hand over to packet scheduler. It might
1062 * immediately call into do_send_packet_in() or it might buffer it for a
1063 * while (until a later call to pinsched_run()). */
1064 pinsched_send(ofconn->schedulers[upcall->type == DPIF_UC_MISS ? 0 : 1],
1065 flow->in_port, ofputil_encode_packet_in(&pin, rw_packet),
1066 do_send_packet_in, ofconn);
1069 /* Fail-open settings. */
1071 /* Returns the failure handling mode (OFPROTO_FAIL_SECURE or
1072 * OFPROTO_FAIL_STANDALONE) for 'mgr'. */
1073 enum ofproto_fail_mode
1074 connmgr_get_fail_mode(const struct connmgr *mgr)
1076 return mgr->fail_mode;
1079 /* Sets the failure handling mode for 'mgr' to 'fail_mode' (either
1080 * OFPROTO_FAIL_SECURE or OFPROTO_FAIL_STANDALONE). */
1082 connmgr_set_fail_mode(struct connmgr *mgr, enum ofproto_fail_mode fail_mode)
1084 mgr->fail_mode = fail_mode;
1085 update_fail_open(mgr);
1088 /* Fail-open implementation. */
1090 /* Returns the longest probe interval among the primary controllers configured
1091 * on 'mgr'. Returns 0 if there are no primary controllers. */
1093 connmgr_get_max_probe_interval(const struct connmgr *mgr)
1095 const struct ofconn *ofconn;
1096 int max_probe_interval;
1098 max_probe_interval = 0;
1099 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1100 int probe_interval = rconn_get_probe_interval(ofconn->rconn);
1101 max_probe_interval = MAX(max_probe_interval, probe_interval);
1103 return max_probe_interval;
1106 /* Returns the number of seconds for which all of 'mgr's primary controllers
1107 * have been disconnected. Returns 0 if 'mgr' has no primary controllers. */
1109 connmgr_failure_duration(const struct connmgr *mgr)
1111 const struct ofconn *ofconn;
1112 int min_failure_duration;
1114 if (!connmgr_has_controllers(mgr)) {
1118 min_failure_duration = INT_MAX;
1119 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1120 int failure_duration = rconn_failure_duration(ofconn->rconn);
1121 min_failure_duration = MIN(min_failure_duration, failure_duration);
1123 return min_failure_duration;
1126 /* Returns true if at least one primary controller is connected (regardless of
1127 * whether those controllers are believed to have authenticated and accepted
1128 * this switch), false if none of them are connected. */
1130 connmgr_is_any_controller_connected(const struct connmgr *mgr)
1132 const struct ofconn *ofconn;
1134 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1135 if (rconn_is_connected(ofconn->rconn)) {
1142 /* Returns true if at least one primary controller is believed to have
1143 * authenticated and accepted this switch, false otherwise. */
1145 connmgr_is_any_controller_admitted(const struct connmgr *mgr)
1147 const struct ofconn *ofconn;
1149 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1150 if (rconn_is_admitted(ofconn->rconn)) {
1157 /* Sends 'packet' to each controller connected to 'mgr'. Takes ownership of
1160 connmgr_broadcast(struct connmgr *mgr, struct ofpbuf *packet)
1162 struct ofconn *ofconn, *prev;
1165 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
1167 ofconn_send_reply(ofconn, ofpbuf_clone(packet));
1169 if (rconn_is_connected(ofconn->rconn)) {
1174 ofconn_send_reply(prev, packet);
1176 ofpbuf_delete(packet);
1180 /* In-band configuration. */
1182 static bool any_extras_changed(const struct connmgr *,
1183 const struct sockaddr_in *extras, size_t n);
1185 /* Sets the 'n' TCP port addresses in 'extras' as ones to which 'mgr''s
1186 * in-band control should guarantee access, in the same way that in-band
1187 * control guarantees access to OpenFlow controllers. */
1189 connmgr_set_extra_in_band_remotes(struct connmgr *mgr,
1190 const struct sockaddr_in *extras, size_t n)
1192 if (!any_extras_changed(mgr, extras, n)) {
1196 free(mgr->extra_in_band_remotes);
1197 mgr->n_extra_remotes = n;
1198 mgr->extra_in_band_remotes = xmemdup(extras, n * sizeof *extras);
1200 update_in_band_remotes(mgr);
1203 /* Sets the OpenFlow queue used by flows set up by in-band control on
1204 * 'mgr' to 'queue_id'. If 'queue_id' is negative, then in-band control
1205 * flows will use the default queue. */
1207 connmgr_set_in_band_queue(struct connmgr *mgr, int queue_id)
1209 if (queue_id != mgr->in_band_queue) {
1210 mgr->in_band_queue = queue_id;
1211 update_in_band_remotes(mgr);
1216 any_extras_changed(const struct connmgr *mgr,
1217 const struct sockaddr_in *extras, size_t n)
1221 if (n != mgr->n_extra_remotes) {
1225 for (i = 0; i < n; i++) {
1226 const struct sockaddr_in *old = &mgr->extra_in_band_remotes[i];
1227 const struct sockaddr_in *new = &extras[i];
1229 if (old->sin_addr.s_addr != new->sin_addr.s_addr ||
1230 old->sin_port != new->sin_port) {
1238 /* In-band implementation. */
1241 connmgr_msg_in_hook(struct connmgr *mgr, const struct flow *flow,
1242 const struct ofpbuf *packet)
1244 return mgr->in_band && in_band_msg_in_hook(mgr->in_band, flow, packet);
1248 connmgr_may_set_up_flow(struct connmgr *mgr, const struct flow *flow,
1249 const struct nlattr *odp_actions,
1252 return !mgr->in_band || in_band_rule_check(flow, odp_actions, actions_len);
1255 /* Fail-open and in-band implementation. */
1257 /* Called by 'ofproto' after all flows have been flushed, to allow fail-open
1258 * and in-band control to re-create their flows. */
1260 connmgr_flushed(struct connmgr *mgr)
1263 in_band_flushed(mgr->in_band);
1265 if (mgr->fail_open) {
1266 fail_open_flushed(mgr->fail_open);
1270 /* Creates a new ofservice for 'target' in 'mgr'. Returns 0 if successful,
1271 * otherwise a positive errno value.
1273 * ofservice_reconfigure() must be called to fully configure the new
1276 ofservice_create(struct connmgr *mgr, const char *target)
1278 struct ofservice *ofservice;
1279 struct pvconn *pvconn;
1282 error = pvconn_open(target, &pvconn);
1287 ofservice = xzalloc(sizeof *ofservice);
1288 hmap_insert(&mgr->services, &ofservice->node, hash_string(target, 0));
1289 ofservice->pvconn = pvconn;
1295 ofservice_destroy(struct connmgr *mgr, struct ofservice *ofservice)
1297 hmap_remove(&mgr->services, &ofservice->node);
1298 pvconn_close(ofservice->pvconn);
1303 ofservice_reconfigure(struct ofservice *ofservice,
1304 const struct ofproto_controller *c)
1306 ofservice->probe_interval = c->probe_interval;
1307 ofservice->rate_limit = c->rate_limit;
1308 ofservice->burst_limit = c->burst_limit;
1311 /* Finds and returns the ofservice within 'mgr' that has the given
1312 * 'target', or a null pointer if none exists. */
1313 static struct ofservice *
1314 ofservice_lookup(struct connmgr *mgr, const char *target)
1316 struct ofservice *ofservice;
1318 HMAP_FOR_EACH_WITH_HASH (ofservice, node, hash_string(target, 0),
1320 if (!strcmp(pvconn_get_name(ofservice->pvconn), target)) {