1 /* Copyright (c) 2009 Nicira Networks
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
18 #include <arpa/inet.h>
28 #include "openflow/nicira-ext.h"
29 #include "openflow/openflow.h"
30 #include "openflow/openflow-mgmt.h"
32 #include "ovs-vswitchd.h"
37 #include "vconn-ssl.h"
38 #include "xenserver.h"
41 #define THIS_MODULE VLM_mgmt
44 #define MAX_BACKOFF_DEFAULT 15
45 #define INACTIVITY_PROBE_DEFAULT 15
47 static struct svec mgmt_cfg;
48 static uint8_t cfg_cookie[CFG_COOKIE_LEN];
49 static struct rconn *mgmt_rconn;
50 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(60, 60);
51 static struct svec capabilities;
52 static struct ofpbuf ext_data_buffer;
56 #define TXQ_LIMIT 128 /* Max number of packets to queue for tx. */
57 struct rconn_packet_counter *txqlen; /* # pkts queued for tx on mgmt_rconn. */
59 static uint64_t pick_fallback_mgmt_id(void);
60 static void send_config_update(uint32_t xid, bool use_xid);
61 static void send_resources_update(uint32_t xid, bool use_xid);
62 static int recv_ofmp(uint32_t xid, struct ofmp_header *ofmph, size_t len);
67 txqlen = rconn_packet_counter_create();
70 svec_init(&capabilities);
71 svec_add_nocopy(&capabilities,
72 xasprintf("com.nicira.mgmt.manager=true\n"));
74 mgmt_id = cfg_get_dpid(0, "mgmt.id");
76 /* Randomly generate a mgmt id */
77 mgmt_id = pick_fallback_mgmt_id();
80 ofpbuf_init(&ext_data_buffer, 0);
85 config_string_change(const char *key, char **valuep)
87 const char *value = cfg_get_string(0, "%s", key);
88 if (value && (!*valuep || strcmp(value, *valuep))) {
90 *valuep = xstrdup(value);
98 mgmt_configure_ssl(void)
100 static char *private_key_file;
101 static char *certificate_file;
102 static char *cacert_file;
104 /* XXX SSL should be configurable separate from the bridges.
105 * XXX should be possible to de-configure SSL. */
106 if (config_string_change("ssl.private-key", &private_key_file)) {
107 vconn_ssl_set_private_key_file(private_key_file);
110 if (config_string_change("ssl.certificate", &certificate_file)) {
111 vconn_ssl_set_certificate_file(certificate_file);
114 if (config_string_change("ssl.ca-cert", &cacert_file)) {
115 vconn_ssl_set_ca_cert_file(cacert_file,
116 cfg_get_bool(0, "ssl.bootstrap-ca-cert"));
122 mgmt_reconfigure(void)
125 uint8_t new_cookie[CFG_COOKIE_LEN];
126 bool cfg_updated = false;
127 const char *controller_name;
129 int inactivity_probe;
132 if (!cfg_has_section("mgmt")) {
134 rconn_destroy(mgmt_rconn);
140 /* If this is an established connection, send a resources update. */
141 /* xxx This is wasteful if there were no resource changes!!! */
143 send_resources_update(0, false);
146 cfg_get_cookie(new_cookie);
147 if (memcmp(cfg_cookie, new_cookie, sizeof(cfg_cookie))) {
148 memcpy(cfg_cookie, new_cookie, sizeof(cfg_cookie));
153 cfg_get_section(&new_cfg, "mgmt");
154 if (svec_equal(&mgmt_cfg, &new_cfg)) {
155 /* Reconnecting to the controller causes the config file to be
156 * resent automatically. If we're not reconnecting and the
157 * config file has changed, we need to notify the controller of
159 if (cfg_updated && mgmt_rconn) {
160 send_config_update(0, false);
162 svec_destroy(&new_cfg);
166 controller_name = cfg_get_string(0, "mgmt.controller");
167 if (!controller_name) {
168 VLOG_ERR("no controller specified for managment");
169 svec_destroy(&new_cfg);
173 max_backoff = cfg_get_int(0, "mgmt.max-backoff");
174 if (max_backoff < 1) {
175 max_backoff = MAX_BACKOFF_DEFAULT;
176 } else if (max_backoff > 3600) {
180 inactivity_probe = cfg_get_int(0, "mgmt.inactivity-probe");
181 if (inactivity_probe < 5) {
182 inactivity_probe = INACTIVITY_PROBE_DEFAULT;
185 /* xxx If this changes, we need to restart bridges to use new id,
186 * xxx but they need the id before the connect to controller, but we
187 * xxx need their dpids. */
188 /* Check if a different mgmt id has been assigned. */
189 if (cfg_has("mgmt.id")) {
190 uint64_t cfg_mgmt_id = cfg_get_dpid(0, "mgmt.id");
191 if (cfg_mgmt_id != mgmt_id) {
192 mgmt_id = cfg_mgmt_id;
196 svec_swap(&new_cfg, &mgmt_cfg);
197 svec_destroy(&new_cfg);
201 mgmt_configure_ssl();
205 rconn_destroy(mgmt_rconn);
208 mgmt_rconn = rconn_create(inactivity_probe, max_backoff);
209 retval = rconn_connect(mgmt_rconn, controller_name);
210 if (retval == EAFNOSUPPORT) {
211 VLOG_ERR("no support for %s vconn", controller_name);
216 make_ofmp_xid(size_t ofmp_len, uint16_t type, uint32_t xid,
217 struct ofpbuf **bufferp)
219 struct ofmp_header *oh;
221 oh = make_openflow_xid(ofmp_len, OFPT_VENDOR, xid, bufferp);
222 oh->header.vendor = htonl(NX_VENDOR_ID);
223 oh->header.subtype = htonl(NXT_MGMT);
224 oh->type = htons(type);
230 make_ofmp(size_t ofmp_len, uint16_t type, struct ofpbuf **bufferp)
232 struct ofmp_header *oh;
234 oh = make_openflow(ofmp_len, OFPT_VENDOR, bufferp);
235 oh->header.vendor = htonl(NX_VENDOR_ID);
236 oh->header.subtype = htonl(NXT_MGMT);
237 oh->type = htons(type);
243 send_openflow_buffer(struct ofpbuf *buffer)
248 VLOG_ERR("attempt to send openflow packet with no rconn\n");
252 /* OpenFlow messages use a 16-bit length field, so messages over 64K
253 * must be broken into multiple pieces.
255 if (buffer->size <= 65535) {
256 update_openflow_length(buffer);
257 retval = rconn_send_with_limit(mgmt_rconn, buffer, txqlen, TXQ_LIMIT);
259 VLOG_WARN_RL(&rl, "send to %s failed: %s",
260 rconn_get_name(mgmt_rconn), strerror(retval));
264 struct ofmp_header *header = (struct ofmp_header *)buffer->data;
265 uint32_t xid = header->header.header.xid;
266 size_t remain = buffer->size;
267 uint8_t *ptr = buffer->data;
269 /* Mark the OpenFlow header with a zero length to indicate some
272 header->header.header.length = 0;
275 struct ofpbuf *new_buffer;
276 struct ofmp_extended_data *oed;
277 size_t new_len = MIN(65535 - sizeof *oed, remain);
279 oed = make_ofmp_xid(sizeof *oed, OFMPT_EXTENDED_DATA, xid,
281 oed->type = header->type;
283 if (remain > 65535) {
284 oed->flags |= OFMPEDF_MORE_DATA;
287 printf("xxx SENDING LEN: %d\n", new_len);
289 /* Copy the entire original message, including the OpenFlow
290 * header, since management protocol structure definitions
291 * include these headers.
293 ofpbuf_put(new_buffer, ptr, new_len);
295 update_openflow_length(new_buffer);
296 retval = rconn_send_with_limit(mgmt_rconn, new_buffer, txqlen,
299 VLOG_WARN_RL(&rl, "send to %s failed: %s",
300 rconn_get_name(mgmt_rconn), strerror(retval));
301 ofpbuf_delete(buffer);
309 ofpbuf_delete(buffer);
315 send_features_reply(uint32_t xid)
317 struct ofpbuf *buffer;
318 struct ofp_switch_features *ofr;
320 ofr = make_openflow_xid(sizeof *ofr, OFPT_FEATURES_REPLY, xid, &buffer);
321 ofr->datapath_id = 0;
324 ofr->capabilities = 0;
326 send_openflow_buffer(buffer);
330 send_capability_reply(uint32_t xid)
333 struct ofpbuf *buffer;
334 struct ofmp_capability_reply *ofmpcr;
336 ofmpcr = make_ofmp_xid(sizeof *ofmpcr, OFMPT_CAPABILITY_REPLY,
338 ofmpcr->format = htonl(OFMPCOF_SIMPLE);
339 ofmpcr->mgmt_id = htonll(mgmt_id);
340 for (i=0; i<capabilities.n; i++) {
341 ofpbuf_put(buffer, capabilities.names[i],
342 strlen(capabilities.names[i]));
344 send_openflow_buffer(buffer);
348 send_resources_update(uint32_t xid, bool use_xid)
350 struct ofpbuf *buffer;
351 struct ofmp_resources_update *ofmpru;
352 struct ofmp_tlv *tlv;
354 struct svec port_list;
355 const char *host_uuid;
359 ofmpru = make_ofmp_xid(sizeof *ofmpru, OFMPT_RESOURCES_UPDATE,
362 ofmpru = make_ofmp(sizeof *ofmpru, OFMPT_RESOURCES_UPDATE, &buffer);
365 /* On XenServer systems, each host has its own UUID, which we provide
368 host_uuid = xenserver_get_host_uuid();
370 struct ofmptsr_mgmt_uuid *mgmt_uuid_tlv;
372 mgmt_uuid_tlv = ofpbuf_put_zeros(buffer, sizeof(*mgmt_uuid_tlv));
373 mgmt_uuid_tlv->type = htons(OFMPTSR_MGMT_UUID);
374 mgmt_uuid_tlv->len = htons(sizeof(*mgmt_uuid_tlv));
375 mgmt_uuid_tlv->mgmt_id = htonll(mgmt_id);
376 memcpy(mgmt_uuid_tlv->uuid, host_uuid, OFMP_UUID_LEN);
380 cfg_get_subsections(&br_list, "bridge");
381 for (i=0; i < br_list.n; i++) {
382 struct ofmptsr_dp *dp_tlv;
386 dp_id = bridge_get_datapathid(br_list.names[i]);
388 VLOG_WARN_RL(&rl, "bridge %s doesn't seem to exist",
392 dp_tlv = ofpbuf_put_zeros(buffer, sizeof(*dp_tlv));
393 dp_tlv->type = htons(OFMPTSR_DP);
394 dp_tlv->len = htons(sizeof(*dp_tlv));
396 dp_tlv->dp_id = htonll(dp_id);
397 memcpy(dp_tlv->name, br_list.names[i], strlen(br_list.names[i])+1);
399 /* On XenServer systems, each network has one or more UUIDs
400 * associated with it, which we provide to the controller.
402 n_uuid = cfg_count("bridge.%s.xs-network-uuids", br_list.names[i]);
404 struct ofmptsr_dp_uuid *dp_uuid_tlv;
405 size_t tlv_len = sizeof(*dp_uuid_tlv) + n_uuid * OFMP_UUID_LEN;
408 dp_uuid_tlv = ofpbuf_put_zeros(buffer, sizeof(*dp_uuid_tlv));
409 dp_uuid_tlv->type = htons(OFMPTSR_DP_UUID);
410 dp_uuid_tlv->len = htons(tlv_len);
411 dp_uuid_tlv->dp_id = htonll(dp_id);
413 for (j=0; j<n_uuid; j++) {
414 const char *dp_uuid = cfg_get_string(j,
415 "bridge.%s.xs-network-uuids", br_list.names[i]);
417 /* The UUID list could change underneath us, so just
418 * fill with zeros in that case. Another update will be
419 * initiated shortly, which should contain corrected data.
422 ofpbuf_put(buffer, dp_uuid, OFMP_UUID_LEN);
424 ofpbuf_put_zeros(buffer, OFMP_UUID_LEN);
429 svec_destroy(&br_list);
431 /* On XenServer systems, extended information about virtual interfaces
432 * (VIFs) is available, which is needed by the controller.
434 svec_init(&port_list);
435 bridge_get_ifaces(&port_list);
436 for (i=0; i < port_list.n; i++) {
437 const char *vif_uuid, *vm_uuid, *net_uuid;
439 struct ofmptsr_vif *vif_tlv;
441 vif_uuid = cfg_get_string(0, "port.%s.vif-uuid", port_list.names[i]);
446 vif_tlv = ofpbuf_put_zeros(buffer, sizeof(*vif_tlv));
447 vif_tlv->type = htons(OFMPTSR_VIF);
448 vif_tlv->len = htons(sizeof(*vif_tlv));
450 memcpy(vif_tlv->name, port_list.names[i], strlen(port_list.names[i])+1);
451 memcpy(vif_tlv->vif_uuid, vif_uuid, sizeof(vif_tlv->vif_uuid));
453 vm_uuid = cfg_get_string(0, "port.%s.vm-uuid", port_list.names[i]);
455 memcpy(vif_tlv->vm_uuid, vm_uuid, sizeof(vif_tlv->vm_uuid));
457 /* In case the vif disappeared underneath us. */
458 memset(vif_tlv->vm_uuid, '\0', sizeof(vif_tlv->vm_uuid));
461 net_uuid = cfg_get_string(0, "port.%s.net-uuid", port_list.names[i]);
463 memcpy(vif_tlv->net_uuid, net_uuid, sizeof(vif_tlv->net_uuid));
465 /* In case the vif disappeared underneath us. */
466 memset(vif_tlv->net_uuid, '\0', sizeof(vif_tlv->net_uuid));
469 vif_mac = cfg_get_mac(0, "port.%s.vif-mac", port_list.names[i]);
470 vif_tlv->vif_mac = htonll(vif_mac);
472 svec_destroy(&port_list);
474 /* Put end marker. */
475 tlv = ofpbuf_put_zeros(buffer, sizeof(*tlv));
476 tlv->type = htons(OFMPTSR_END);
477 tlv->len = htons(sizeof(*tlv));
478 send_openflow_buffer(buffer);
482 send_config_update(uint32_t xid, bool use_xid)
484 struct ofpbuf *buffer;
485 struct ofmp_config_update *ofmpcu;
488 ofmpcu = make_ofmp_xid(sizeof *ofmpcu, OFMPT_CONFIG_UPDATE,
491 ofmpcu = make_ofmp(sizeof *ofmpcu, OFMPT_CONFIG_UPDATE, &buffer);
494 ofmpcu->format = htonl(OFMPCOF_SIMPLE);
495 memcpy(ofmpcu->cookie, cfg_cookie, sizeof(ofmpcu->cookie));
497 send_openflow_buffer(buffer);
501 send_config_update_ack(uint32_t xid, bool success)
503 struct ofpbuf *buffer;
504 struct ofmp_config_update_ack *ofmpcua;
506 ofmpcua = make_ofmp_xid(sizeof *ofmpcua, OFMPT_CONFIG_UPDATE_ACK,
509 ofmpcua->format = htonl(OFMPCOF_SIMPLE);
511 ofmpcua->flags = htonl(OFMPCUAF_SUCCESS);
513 cfg_get_cookie(ofmpcua->cookie);
514 send_openflow_buffer(buffer);
518 send_ofmp_error_msg(uint32_t xid, uint16_t type, uint16_t code,
519 const void *data, size_t len)
521 struct ofpbuf *buffer;
522 struct ofmp_error_msg *oem;
524 oem = make_ofmp_xid(sizeof(*oem)+len, OFMPT_ERROR, xid, &buffer);
525 oem->type = htons(type);
526 oem->code = htons(code);
527 memcpy(oem->data, data, len);
528 send_openflow_buffer(buffer);
532 send_error_msg(uint32_t xid, uint16_t type, uint16_t code,
533 const void *data, size_t len)
535 struct ofpbuf *buffer;
536 struct ofp_error_msg *oem;
538 oem = make_openflow_xid(sizeof(*oem)+len, OFPT_ERROR, xid, &buffer);
539 oem->type = htons(type);
540 oem->code = htons(code);
541 memcpy(oem->data, data, len);
542 send_openflow_buffer(buffer);
546 recv_echo_request(uint32_t xid UNUSED, const void *msg)
548 const struct ofp_header *rq = msg;
549 send_openflow_buffer(make_echo_reply(rq));
554 recv_features_request(uint32_t xid, const void *msg UNUSED)
556 send_features_reply(xid);
561 recv_set_config(uint32_t xid UNUSED, const void *msg UNUSED)
563 /* Nothing to configure! */
568 recv_ofmp_capability_request(uint32_t xid, const struct ofmp_header *ofmph,
571 struct ofmp_capability_request *ofmpcr;
573 if (len != sizeof(*ofmpcr)) {
578 ofmpcr = (struct ofmp_capability_request *)ofmph;
579 if (ofmpcr->format != htonl(OFMPCAF_SIMPLE)) {
584 send_capability_reply(xid);
590 recv_ofmp_resources_request(uint32_t xid, const void *msg UNUSED,
593 send_resources_update(xid, true);
598 recv_ofmp_config_request(uint32_t xid, const struct ofmp_header *ofmph,
601 struct ofmp_config_request *ofmpcr;
603 if (len != sizeof(*ofmpcr)) {
608 ofmpcr = (struct ofmp_config_request *)ofmph;
609 if (ofmpcr->format != htonl(OFMPCOF_SIMPLE)) {
614 send_config_update(xid, true);
620 recv_ofmp_config_update(uint32_t xid, const struct ofmp_header *ofmph,
623 struct ofmp_config_update *ofmpcu;
626 data_len = len - sizeof(*ofmpcu);
627 if (data_len <= sizeof(*ofmpcu)) {
628 /* xxx Send error. */
632 ofmpcu = (struct ofmp_config_update *)ofmph;
633 if (ofmpcu->format != htonl(OFMPCOF_SIMPLE)) {
638 /* Check if the supplied cookie matches our current understanding of
639 * it. If they don't match, tell the controller and let it sort
641 if (cfg_lock(ofmpcu->cookie, 0)) {
642 /* xxx cfg_lock can fail for other reasons, such as being
644 VLOG_WARN_RL(&rl, "config update failed due to bad cookie\n");
645 send_config_update_ack(xid, false);
649 /* xxx We should probably do more sanity checking than this. */
651 cfg_write_data(ofmpcu->data, data_len);
654 /* Send the ACK before running reconfigure, since our management
655 * connection settings may have changed. */
656 send_config_update_ack(xid, true);
665 recv_ofmp_extended_data(uint32_t xid, const struct ofmp_header *ofmph,
669 struct ofmp_extended_data *ofmped;
672 data_len = len - sizeof(*ofmped);
673 if (data_len <= sizeof(*ofmped)) {
674 /* xxx Send error. */
678 ofmped = (struct ofmp_extended_data *)ofmph;
680 ptr = ofpbuf_put(&ext_data_buffer, ofmped->data, data_len);
682 if (!ofmped->flags & OFMPEDF_MORE_DATA) {
683 recv_ofmp(xid, ext_data_buffer.data, ext_data_buffer.size);
684 ofpbuf_clear(&ext_data_buffer);
690 /* Handles receiving a management message. Generally, this function
691 * will be called 'len' set to zero, and the length will be derived by
692 * the OpenFlow header. With the extended data message, management
693 * messages are not constrained by OpenFlow's 64K message length limit.
694 * The extended data handler calls this function with the 'len' set to
695 * the total message length and the OpenFlow header's length field is
699 int recv_ofmp(uint32_t xid, struct ofmp_header *ofmph, size_t len)
702 len = ntohs(ofmph->header.header.length);
705 /* xxx Should sanity-check for min/max length */
706 switch (ntohs(ofmph->type))
708 case OFMPT_CAPABILITY_REQUEST:
709 return recv_ofmp_capability_request(xid, ofmph, len);
710 case OFMPT_RESOURCES_REQUEST:
711 return recv_ofmp_resources_request(xid, ofmph, len);
712 case OFMPT_CONFIG_REQUEST:
713 return recv_ofmp_config_request(xid, ofmph, len);
714 case OFMPT_CONFIG_UPDATE:
715 return recv_ofmp_config_update(xid, ofmph, len);
716 case OFMPT_EXTENDED_DATA:
717 return recv_ofmp_extended_data(xid, ofmph, len);
719 VLOG_WARN_RL(&rl, "unknown mgmt message: %d",
726 recv_nx_msg(uint32_t xid, const void *oh)
728 const struct nicira_header *nh = oh;
730 switch (ntohl(nh->subtype)) {
733 return recv_ofmp(xid, (struct ofmp_header *)oh, 0);
736 send_error_msg(xid, OFPET_BAD_REQUEST, OFPBRC_BAD_SUBTYPE,
737 oh, ntohs(nh->header.length));
743 recv_vendor(uint32_t xid, const void *oh)
745 const struct ofp_vendor_header *ovh = oh;
747 switch (ntohl(ovh->vendor))
750 return recv_nx_msg(xid, oh);
753 VLOG_WARN_RL(&rl, "unknown vendor: 0x%x", ntohl(ovh->vendor));
754 send_error_msg(xid, OFPET_BAD_REQUEST, OFPBRC_BAD_VENDOR,
755 oh, ntohs(ovh->header.length));
761 handle_msg(uint32_t xid, const void *msg, size_t length)
763 int (*handler)(uint32_t, const void *);
764 struct ofp_header *oh;
767 COVERAGE_INC(mgmt_received);
769 /* Check encapsulated length. */
770 oh = (struct ofp_header *) msg;
771 if (ntohs(oh->length) > length) {
774 assert(oh->version == OFP_VERSION);
776 /* Figure out how to handle it. */
778 case OFPT_ECHO_REQUEST:
779 min_size = sizeof(struct ofp_header);
780 handler = recv_echo_request;
782 case OFPT_ECHO_REPLY:
784 case OFPT_FEATURES_REQUEST:
785 min_size = sizeof(struct ofp_header);
786 handler = recv_features_request;
788 case OFPT_SET_CONFIG:
789 min_size = sizeof(struct ofp_switch_config);
790 handler = recv_set_config;
793 min_size = sizeof(struct ofp_vendor_header);
794 handler = recv_vendor;
797 VLOG_WARN_RL(&rl, "unknown openflow type: %d", oh->type);
798 send_error_msg(xid, OFPET_BAD_REQUEST, OFPBRC_BAD_TYPE,
804 if (length < min_size) {
807 return handler(xid, msg);
819 rconn_run(mgmt_rconn);
821 /* Do some processing, but cap it at a reasonable amount so that
822 * other processing doesn't starve. */
823 for (i=0; i<50; i++) {
824 struct ofpbuf *buffer;
825 struct ofp_header *oh;
827 buffer = rconn_recv(mgmt_rconn);
832 if (buffer->size >= sizeof *oh) {
834 handle_msg(oh->xid, buffer->data, buffer->size);
835 ofpbuf_delete(buffer);
837 VLOG_WARN_RL(&rl, "received too-short OpenFlow message");
849 rconn_run_wait(mgmt_rconn);
850 rconn_recv_wait(mgmt_rconn);
854 pick_fallback_mgmt_id(void)
856 uint8_t ea[ETH_ADDR_LEN];
858 ea[0] = 0x00; /* Set Nicira OUI. */
861 return eth_addr_to_uint64(ea);