1 /* Copyright (c) 2009 Nicira Networks
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
18 #include <arpa/inet.h>
28 #include "openflow/nicira-ext.h"
29 #include "openflow/openflow.h"
30 #include "openflow/openflow-mgmt.h"
32 #include "ovs-vswitchd.h"
37 #include "vconn-ssl.h"
38 #include "xenserver.h"
41 #define THIS_MODULE VLM_mgmt
44 #define MAX_BACKOFF_DEFAULT 15
45 #define INACTIVITY_PROBE_DEFAULT 15
47 static struct svec mgmt_cfg;
48 static uint8_t cfg_cookie[CFG_COOKIE_LEN];
49 static bool need_reconfigure = false;
50 static struct rconn *mgmt_rconn;
51 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(60, 60);
52 static struct svec capabilities;
53 static struct ofpbuf ext_data_buffer;
57 #define TXQ_LIMIT 128 /* Max number of packets to queue for tx. */
58 struct rconn_packet_counter *txqlen; /* # pkts queued for tx on mgmt_rconn. */
60 static uint64_t pick_fallback_mgmt_id(void);
61 static void send_config_update(uint32_t xid, bool use_xid);
62 static void send_resources_update(uint32_t xid, bool use_xid);
63 static int recv_ofmp(uint32_t xid, struct ofmp_header *ofmph, size_t len);
68 txqlen = rconn_packet_counter_create();
71 svec_init(&capabilities);
72 svec_add_nocopy(&capabilities,
73 xasprintf("com.nicira.mgmt.manager=true\n"));
75 mgmt_id = cfg_get_dpid(0, "mgmt.id");
77 /* Randomly generate a mgmt id */
78 mgmt_id = pick_fallback_mgmt_id();
81 ofpbuf_init(&ext_data_buffer, 0);
86 config_string_change(const char *key, char **valuep)
88 const char *value = cfg_get_string(0, "%s", key);
89 if (value && (!*valuep || strcmp(value, *valuep))) {
91 *valuep = xstrdup(value);
99 mgmt_configure_ssl(void)
101 static char *private_key_file;
102 static char *certificate_file;
103 static char *cacert_file;
105 /* XXX SSL should be configurable separate from the bridges.
106 * XXX should be possible to de-configure SSL. */
107 if (config_string_change("ssl.private-key", &private_key_file)) {
108 vconn_ssl_set_private_key_file(private_key_file);
111 if (config_string_change("ssl.certificate", &certificate_file)) {
112 vconn_ssl_set_certificate_file(certificate_file);
115 if (config_string_change("ssl.ca-cert", &cacert_file)) {
116 vconn_ssl_set_ca_cert_file(cacert_file,
117 cfg_get_bool(0, "ssl.bootstrap-ca-cert"));
123 mgmt_reconfigure(void)
126 uint8_t new_cookie[CFG_COOKIE_LEN];
127 bool cfg_updated = false;
128 const char *controller_name;
130 int inactivity_probe;
133 if (!cfg_has_section("mgmt")) {
134 svec_clear(&mgmt_cfg);
136 rconn_destroy(mgmt_rconn);
142 /* If this is an established connection, send a resources update. */
143 /* xxx This is wasteful if there were no resource changes!!! */
145 send_resources_update(0, false);
148 cfg_get_cookie(new_cookie);
149 if (memcmp(cfg_cookie, new_cookie, sizeof(cfg_cookie))) {
150 memcpy(cfg_cookie, new_cookie, sizeof(cfg_cookie));
155 cfg_get_section(&new_cfg, "mgmt");
156 if (svec_equal(&mgmt_cfg, &new_cfg)) {
157 /* Reconnecting to the controller causes the config file to be
158 * resent automatically. If we're not reconnecting and the
159 * config file has changed, we need to notify the controller of
161 if (cfg_updated && mgmt_rconn) {
162 send_config_update(0, false);
164 svec_destroy(&new_cfg);
168 controller_name = cfg_get_string(0, "mgmt.controller");
169 if (!controller_name) {
170 VLOG_ERR("no controller specified for managment");
171 svec_destroy(&new_cfg);
175 max_backoff = cfg_get_int(0, "mgmt.max-backoff");
176 if (max_backoff < 1) {
177 max_backoff = MAX_BACKOFF_DEFAULT;
178 } else if (max_backoff > 3600) {
182 inactivity_probe = cfg_get_int(0, "mgmt.inactivity-probe");
183 if (inactivity_probe < 5) {
184 inactivity_probe = INACTIVITY_PROBE_DEFAULT;
187 /* xxx If this changes, we need to restart bridges to use new id,
188 * xxx but they need the id before the connect to controller, but we
189 * xxx need their dpids. */
190 /* Check if a different mgmt id has been assigned. */
191 if (cfg_has("mgmt.id")) {
192 uint64_t cfg_mgmt_id = cfg_get_dpid(0, "mgmt.id");
193 if (cfg_mgmt_id != mgmt_id) {
194 mgmt_id = cfg_mgmt_id;
198 svec_swap(&new_cfg, &mgmt_cfg);
199 svec_destroy(&new_cfg);
203 mgmt_configure_ssl();
207 rconn_destroy(mgmt_rconn);
210 mgmt_rconn = rconn_create(inactivity_probe, max_backoff);
211 retval = rconn_connect(mgmt_rconn, controller_name);
212 if (retval == EAFNOSUPPORT) {
213 VLOG_ERR("no support for %s vconn", controller_name);
218 make_ofmp_xid(size_t ofmp_len, uint16_t type, uint32_t xid,
219 struct ofpbuf **bufferp)
221 struct ofmp_header *oh;
223 oh = make_openflow_xid(ofmp_len, OFPT_VENDOR, xid, bufferp);
224 oh->header.vendor = htonl(NX_VENDOR_ID);
225 oh->header.subtype = htonl(NXT_MGMT);
226 oh->type = htons(type);
232 make_ofmp(size_t ofmp_len, uint16_t type, struct ofpbuf **bufferp)
234 struct ofmp_header *oh;
236 oh = make_openflow(ofmp_len, OFPT_VENDOR, bufferp);
237 oh->header.vendor = htonl(NX_VENDOR_ID);
238 oh->header.subtype = htonl(NXT_MGMT);
239 oh->type = htons(type);
245 send_openflow_buffer(struct ofpbuf *buffer)
250 VLOG_ERR("attempt to send openflow packet with no rconn\n");
254 /* OpenFlow messages use a 16-bit length field, so messages over 64K
255 * must be broken into multiple pieces.
257 if (buffer->size <= 65535) {
258 update_openflow_length(buffer);
259 retval = rconn_send_with_limit(mgmt_rconn, buffer, txqlen, TXQ_LIMIT);
261 VLOG_WARN_RL(&rl, "send to %s failed: %s",
262 rconn_get_name(mgmt_rconn), strerror(retval));
266 struct ofmp_header *header = (struct ofmp_header *)buffer->data;
267 uint32_t xid = header->header.header.xid;
268 size_t remain = buffer->size;
269 uint8_t *ptr = buffer->data;
271 /* Mark the OpenFlow header with a zero length to indicate some
274 header->header.header.length = 0;
277 struct ofpbuf *new_buffer;
278 struct ofmp_extended_data *oed;
279 size_t new_len = MIN(65535 - sizeof *oed, remain);
281 oed = make_ofmp_xid(sizeof *oed, OFMPT_EXTENDED_DATA, xid,
283 oed->type = header->type;
285 if (remain > 65535) {
286 oed->flags |= OFMPEDF_MORE_DATA;
289 printf("xxx SENDING LEN: %d\n", new_len);
291 /* Copy the entire original message, including the OpenFlow
292 * header, since management protocol structure definitions
293 * include these headers.
295 ofpbuf_put(new_buffer, ptr, new_len);
297 update_openflow_length(new_buffer);
298 retval = rconn_send_with_limit(mgmt_rconn, new_buffer, txqlen,
301 VLOG_WARN_RL(&rl, "send to %s failed: %s",
302 rconn_get_name(mgmt_rconn), strerror(retval));
303 ofpbuf_delete(buffer);
311 ofpbuf_delete(buffer);
317 send_features_reply(uint32_t xid)
319 struct ofpbuf *buffer;
320 struct ofp_switch_features *ofr;
322 ofr = make_openflow_xid(sizeof *ofr, OFPT_FEATURES_REPLY, xid, &buffer);
323 ofr->datapath_id = 0;
326 ofr->capabilities = 0;
328 send_openflow_buffer(buffer);
332 send_capability_reply(uint32_t xid)
335 struct ofpbuf *buffer;
336 struct ofmp_capability_reply *ofmpcr;
338 ofmpcr = make_ofmp_xid(sizeof *ofmpcr, OFMPT_CAPABILITY_REPLY,
340 ofmpcr->format = htonl(OFMPCOF_SIMPLE);
341 ofmpcr->mgmt_id = htonll(mgmt_id);
342 for (i=0; i<capabilities.n; i++) {
343 ofpbuf_put(buffer, capabilities.names[i],
344 strlen(capabilities.names[i]));
346 send_openflow_buffer(buffer);
350 send_resources_update(uint32_t xid, bool use_xid)
352 struct ofpbuf *buffer;
353 struct ofmp_resources_update *ofmpru;
354 struct ofmp_tlv *tlv;
356 struct svec port_list;
357 const char *host_uuid;
361 ofmpru = make_ofmp_xid(sizeof *ofmpru, OFMPT_RESOURCES_UPDATE,
364 ofmpru = make_ofmp(sizeof *ofmpru, OFMPT_RESOURCES_UPDATE, &buffer);
367 /* On XenServer systems, each host has its own UUID, which we provide
370 host_uuid = xenserver_get_host_uuid();
372 struct ofmptsr_mgmt_uuid *mgmt_uuid_tlv;
374 mgmt_uuid_tlv = ofpbuf_put_zeros(buffer, sizeof(*mgmt_uuid_tlv));
375 mgmt_uuid_tlv->type = htons(OFMPTSR_MGMT_UUID);
376 mgmt_uuid_tlv->len = htons(sizeof(*mgmt_uuid_tlv));
377 mgmt_uuid_tlv->mgmt_id = htonll(mgmt_id);
378 memcpy(mgmt_uuid_tlv->uuid, host_uuid, OFMP_UUID_LEN);
382 cfg_get_subsections(&br_list, "bridge");
383 for (i=0; i < br_list.n; i++) {
384 struct ofmptsr_dp *dp_tlv;
388 dp_id = bridge_get_datapathid(br_list.names[i]);
390 VLOG_WARN_RL(&rl, "bridge %s doesn't seem to exist",
394 dp_tlv = ofpbuf_put_zeros(buffer, sizeof(*dp_tlv));
395 dp_tlv->type = htons(OFMPTSR_DP);
396 dp_tlv->len = htons(sizeof(*dp_tlv));
398 dp_tlv->dp_id = htonll(dp_id);
399 memcpy(dp_tlv->name, br_list.names[i], strlen(br_list.names[i])+1);
401 /* On XenServer systems, each network has one or more UUIDs
402 * associated with it, which we provide to the controller.
404 n_uuid = cfg_count("bridge.%s.xs-network-uuids", br_list.names[i]);
406 struct ofmptsr_dp_uuid *dp_uuid_tlv;
407 size_t tlv_len = sizeof(*dp_uuid_tlv) + n_uuid * OFMP_UUID_LEN;
410 dp_uuid_tlv = ofpbuf_put_zeros(buffer, sizeof(*dp_uuid_tlv));
411 dp_uuid_tlv->type = htons(OFMPTSR_DP_UUID);
412 dp_uuid_tlv->len = htons(tlv_len);
413 dp_uuid_tlv->dp_id = htonll(dp_id);
415 for (j=0; j<n_uuid; j++) {
416 const char *dp_uuid = cfg_get_string(j,
417 "bridge.%s.xs-network-uuids", br_list.names[i]);
419 /* The UUID list could change underneath us, so just
420 * fill with zeros in that case. Another update will be
421 * initiated shortly, which should contain corrected data.
424 ofpbuf_put(buffer, dp_uuid, OFMP_UUID_LEN);
426 ofpbuf_put_zeros(buffer, OFMP_UUID_LEN);
431 svec_destroy(&br_list);
433 /* On XenServer systems, extended information about virtual interfaces
434 * (VIFs) is available, which is needed by the controller.
436 svec_init(&port_list);
437 bridge_get_ifaces(&port_list);
438 for (i=0; i < port_list.n; i++) {
439 const char *vif_uuid, *vm_uuid, *net_uuid;
441 struct ofmptsr_vif *vif_tlv;
443 vif_uuid = cfg_get_string(0, "port.%s.vif-uuid", port_list.names[i]);
448 vif_tlv = ofpbuf_put_zeros(buffer, sizeof(*vif_tlv));
449 vif_tlv->type = htons(OFMPTSR_VIF);
450 vif_tlv->len = htons(sizeof(*vif_tlv));
452 memcpy(vif_tlv->name, port_list.names[i], strlen(port_list.names[i])+1);
453 memcpy(vif_tlv->vif_uuid, vif_uuid, sizeof(vif_tlv->vif_uuid));
455 vm_uuid = cfg_get_string(0, "port.%s.vm-uuid", port_list.names[i]);
457 memcpy(vif_tlv->vm_uuid, vm_uuid, sizeof(vif_tlv->vm_uuid));
459 /* In case the vif disappeared underneath us. */
460 memset(vif_tlv->vm_uuid, '\0', sizeof(vif_tlv->vm_uuid));
463 net_uuid = cfg_get_string(0, "port.%s.net-uuid", port_list.names[i]);
465 memcpy(vif_tlv->net_uuid, net_uuid, sizeof(vif_tlv->net_uuid));
467 /* In case the vif disappeared underneath us. */
468 memset(vif_tlv->net_uuid, '\0', sizeof(vif_tlv->net_uuid));
471 vif_mac = cfg_get_mac(0, "port.%s.vif-mac", port_list.names[i]);
472 vif_tlv->vif_mac = htonll(vif_mac);
474 svec_destroy(&port_list);
476 /* Put end marker. */
477 tlv = ofpbuf_put_zeros(buffer, sizeof(*tlv));
478 tlv->type = htons(OFMPTSR_END);
479 tlv->len = htons(sizeof(*tlv));
480 send_openflow_buffer(buffer);
484 send_config_update(uint32_t xid, bool use_xid)
486 struct ofpbuf *buffer;
487 struct ofmp_config_update *ofmpcu;
490 ofmpcu = make_ofmp_xid(sizeof *ofmpcu, OFMPT_CONFIG_UPDATE,
493 ofmpcu = make_ofmp(sizeof *ofmpcu, OFMPT_CONFIG_UPDATE, &buffer);
496 ofmpcu->format = htonl(OFMPCOF_SIMPLE);
497 memcpy(ofmpcu->cookie, cfg_cookie, sizeof(ofmpcu->cookie));
499 send_openflow_buffer(buffer);
503 send_config_update_ack(uint32_t xid, bool success)
505 struct ofpbuf *buffer;
506 struct ofmp_config_update_ack *ofmpcua;
508 ofmpcua = make_ofmp_xid(sizeof *ofmpcua, OFMPT_CONFIG_UPDATE_ACK,
511 ofmpcua->format = htonl(OFMPCOF_SIMPLE);
513 ofmpcua->flags = htonl(OFMPCUAF_SUCCESS);
515 cfg_get_cookie(ofmpcua->cookie);
516 send_openflow_buffer(buffer);
520 send_ofmp_error_msg(uint32_t xid, uint16_t type, uint16_t code,
521 const void *data, size_t len)
523 struct ofpbuf *buffer;
524 struct ofmp_error_msg *oem;
526 oem = make_ofmp_xid(sizeof(*oem)+len, OFMPT_ERROR, xid, &buffer);
527 oem->type = htons(type);
528 oem->code = htons(code);
529 memcpy(oem->data, data, len);
530 send_openflow_buffer(buffer);
534 send_error_msg(uint32_t xid, uint16_t type, uint16_t code,
535 const void *data, size_t len)
537 struct ofpbuf *buffer;
538 struct ofp_error_msg *oem;
540 oem = make_openflow_xid(sizeof(*oem)+len, OFPT_ERROR, xid, &buffer);
541 oem->type = htons(type);
542 oem->code = htons(code);
543 memcpy(oem->data, data, len);
544 send_openflow_buffer(buffer);
548 recv_echo_request(uint32_t xid UNUSED, const void *msg)
550 const struct ofp_header *rq = msg;
551 send_openflow_buffer(make_echo_reply(rq));
556 recv_features_request(uint32_t xid, const void *msg UNUSED)
558 send_features_reply(xid);
563 recv_set_config(uint32_t xid UNUSED, const void *msg UNUSED)
565 /* Nothing to configure! */
570 recv_ofmp_capability_request(uint32_t xid, const struct ofmp_header *ofmph,
573 struct ofmp_capability_request *ofmpcr;
575 if (len != sizeof(*ofmpcr)) {
580 ofmpcr = (struct ofmp_capability_request *)ofmph;
581 if (ofmpcr->format != htonl(OFMPCAF_SIMPLE)) {
586 send_capability_reply(xid);
592 recv_ofmp_resources_request(uint32_t xid, const void *msg UNUSED,
595 send_resources_update(xid, true);
600 recv_ofmp_config_request(uint32_t xid, const struct ofmp_header *ofmph,
603 struct ofmp_config_request *ofmpcr;
605 if (len != sizeof(*ofmpcr)) {
610 ofmpcr = (struct ofmp_config_request *)ofmph;
611 if (ofmpcr->format != htonl(OFMPCOF_SIMPLE)) {
616 send_config_update(xid, true);
622 recv_ofmp_config_update(uint32_t xid, const struct ofmp_header *ofmph,
625 struct ofmp_config_update *ofmpcu;
628 data_len = len - sizeof(*ofmpcu);
629 if (data_len <= sizeof(*ofmpcu)) {
630 /* xxx Send error. */
634 ofmpcu = (struct ofmp_config_update *)ofmph;
635 if (ofmpcu->format != htonl(OFMPCOF_SIMPLE)) {
640 /* Check if the supplied cookie matches our current understanding of
641 * it. If they don't match, tell the controller and let it sort
643 if (cfg_lock(ofmpcu->cookie, 0)) {
644 /* xxx cfg_lock can fail for other reasons, such as being
646 VLOG_WARN_RL(&rl, "config update failed due to bad cookie\n");
647 send_config_update_ack(xid, false);
651 /* xxx We should probably do more sanity checking than this. */
653 cfg_write_data(ofmpcu->data, data_len);
656 /* Send the ACK before running reconfigure, since our management
657 * connection settings may have changed. */
658 send_config_update_ack(xid, true);
660 need_reconfigure = true;
666 recv_ofmp_extended_data(uint32_t xid, const struct ofmp_header *ofmph,
670 struct ofmp_extended_data *ofmped;
673 data_len = len - sizeof(*ofmped);
674 if (data_len <= sizeof(*ofmped)) {
675 /* xxx Send error. */
679 ofmped = (struct ofmp_extended_data *)ofmph;
681 ptr = ofpbuf_put(&ext_data_buffer, ofmped->data, data_len);
683 if (!ofmped->flags & OFMPEDF_MORE_DATA) {
684 recv_ofmp(xid, ext_data_buffer.data, ext_data_buffer.size);
685 ofpbuf_clear(&ext_data_buffer);
691 /* Handles receiving a management message. Generally, this function
692 * will be called 'len' set to zero, and the length will be derived by
693 * the OpenFlow header. With the extended data message, management
694 * messages are not constrained by OpenFlow's 64K message length limit.
695 * The extended data handler calls this function with the 'len' set to
696 * the total message length and the OpenFlow header's length field is
700 int recv_ofmp(uint32_t xid, struct ofmp_header *ofmph, size_t len)
703 len = ntohs(ofmph->header.header.length);
706 /* xxx Should sanity-check for min/max length */
707 switch (ntohs(ofmph->type))
709 case OFMPT_CAPABILITY_REQUEST:
710 return recv_ofmp_capability_request(xid, ofmph, len);
711 case OFMPT_RESOURCES_REQUEST:
712 return recv_ofmp_resources_request(xid, ofmph, len);
713 case OFMPT_CONFIG_REQUEST:
714 return recv_ofmp_config_request(xid, ofmph, len);
715 case OFMPT_CONFIG_UPDATE:
716 return recv_ofmp_config_update(xid, ofmph, len);
717 case OFMPT_EXTENDED_DATA:
718 return recv_ofmp_extended_data(xid, ofmph, len);
720 VLOG_WARN_RL(&rl, "unknown mgmt message: %d",
727 recv_nx_msg(uint32_t xid, const void *oh)
729 const struct nicira_header *nh = oh;
731 switch (ntohl(nh->subtype)) {
734 return recv_ofmp(xid, (struct ofmp_header *)oh, 0);
737 send_error_msg(xid, OFPET_BAD_REQUEST, OFPBRC_BAD_SUBTYPE,
738 oh, ntohs(nh->header.length));
744 recv_vendor(uint32_t xid, const void *oh)
746 const struct ofp_vendor_header *ovh = oh;
748 switch (ntohl(ovh->vendor))
751 return recv_nx_msg(xid, oh);
754 VLOG_WARN_RL(&rl, "unknown vendor: 0x%x", ntohl(ovh->vendor));
755 send_error_msg(xid, OFPET_BAD_REQUEST, OFPBRC_BAD_VENDOR,
756 oh, ntohs(ovh->header.length));
762 handle_msg(uint32_t xid, const void *msg, size_t length)
764 int (*handler)(uint32_t, const void *);
765 struct ofp_header *oh;
768 COVERAGE_INC(mgmt_received);
770 /* Check encapsulated length. */
771 oh = (struct ofp_header *) msg;
772 if (ntohs(oh->length) > length) {
775 assert(oh->version == OFP_VERSION);
777 /* Figure out how to handle it. */
779 case OFPT_ECHO_REQUEST:
780 min_size = sizeof(struct ofp_header);
781 handler = recv_echo_request;
783 case OFPT_ECHO_REPLY:
785 case OFPT_FEATURES_REQUEST:
786 min_size = sizeof(struct ofp_header);
787 handler = recv_features_request;
789 case OFPT_SET_CONFIG:
790 min_size = sizeof(struct ofp_switch_config);
791 handler = recv_set_config;
794 min_size = sizeof(struct ofp_vendor_header);
795 handler = recv_vendor;
798 VLOG_WARN_RL(&rl, "unknown openflow type: %d", oh->type);
799 send_error_msg(xid, OFPET_BAD_REQUEST, OFPBRC_BAD_TYPE,
805 if (length < min_size) {
808 return handler(xid, msg);
820 need_reconfigure = false;
821 rconn_run(mgmt_rconn);
823 /* Do some processing, but cap it at a reasonable amount so that
824 * other processing doesn't starve. */
825 for (i=0; i<50; i++) {
826 struct ofpbuf *buffer;
827 struct ofp_header *oh;
829 buffer = rconn_recv(mgmt_rconn);
834 if (buffer->size >= sizeof *oh) {
836 handle_msg(oh->xid, buffer->data, buffer->size);
837 ofpbuf_delete(buffer);
839 VLOG_WARN_RL(&rl, "received too-short OpenFlow message");
843 return need_reconfigure;
853 rconn_run_wait(mgmt_rconn);
854 rconn_recv_wait(mgmt_rconn);
858 pick_fallback_mgmt_id(void)
860 uint8_t ea[ETH_ADDR_LEN];
862 ea[0] = 0x00; /* Set Nicira OUI. */
865 return eth_addr_to_uint64(ea);