1 /* Copyright (c) 2009 Nicira Networks
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
18 #include <arpa/inet.h>
28 #include "openflow/nicira-ext.h"
29 #include "openflow/openflow.h"
30 #include "openflow/openflow-mgmt.h"
32 #include "ovs-vswitchd.h"
37 #include "vconn-ssl.h"
38 #include "xenserver.h"
41 #define THIS_MODULE VLM_mgmt
44 #define MAX_BACKOFF_DEFAULT 15
45 #define INACTIVITY_PROBE_DEFAULT 15
47 static struct svec mgmt_cfg;
48 static uint8_t cfg_cookie[CFG_COOKIE_LEN];
49 static bool need_reconfigure = false;
50 static struct rconn *mgmt_rconn;
51 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(60, 60);
52 static struct svec capabilities;
53 static struct ofpbuf ext_data_buffer;
57 #define TXQ_LIMIT 128 /* Max number of packets to queue for tx. */
58 struct rconn_packet_counter *txqlen; /* # pkts queued for tx on mgmt_rconn. */
60 static uint64_t pick_fallback_mgmt_id(void);
61 static void send_config_update(uint32_t xid, bool use_xid);
62 static void send_resources_update(uint32_t xid, bool use_xid);
63 static int recv_ofmp(uint32_t xid, struct ofmp_header *ofmph, size_t len);
68 txqlen = rconn_packet_counter_create();
71 svec_init(&capabilities);
72 svec_add_nocopy(&capabilities,
73 xasprintf("com.nicira.mgmt.manager=true\n"));
75 mgmt_id = cfg_get_dpid(0, "mgmt.id");
77 /* Randomly generate a mgmt id */
78 mgmt_id = pick_fallback_mgmt_id();
81 ofpbuf_init(&ext_data_buffer, 0);
86 config_string_change(const char *key, char **valuep)
88 const char *value = cfg_get_string(0, "%s", key);
89 if (value && (!*valuep || strcmp(value, *valuep))) {
91 *valuep = xstrdup(value);
99 mgmt_configure_ssl(void)
101 static char *private_key_file;
102 static char *certificate_file;
103 static char *cacert_file;
105 /* XXX SSL should be configurable separate from the bridges.
106 * XXX should be possible to de-configure SSL. */
107 if (config_string_change("ssl.private-key", &private_key_file)) {
108 vconn_ssl_set_private_key_file(private_key_file);
111 if (config_string_change("ssl.certificate", &certificate_file)) {
112 vconn_ssl_set_certificate_file(certificate_file);
115 if (config_string_change("ssl.ca-cert", &cacert_file)) {
116 vconn_ssl_set_ca_cert_file(cacert_file,
117 cfg_get_bool(0, "ssl.bootstrap-ca-cert"));
123 mgmt_reconfigure(void)
126 uint8_t new_cookie[CFG_COOKIE_LEN];
127 bool cfg_updated = false;
128 const char *controller_name;
130 int inactivity_probe;
133 if (!cfg_has_section("mgmt")) {
135 rconn_destroy(mgmt_rconn);
141 /* If this is an established connection, send a resources update. */
142 /* xxx This is wasteful if there were no resource changes!!! */
144 send_resources_update(0, false);
147 cfg_get_cookie(new_cookie);
148 if (memcmp(cfg_cookie, new_cookie, sizeof(cfg_cookie))) {
149 memcpy(cfg_cookie, new_cookie, sizeof(cfg_cookie));
154 cfg_get_section(&new_cfg, "mgmt");
155 if (svec_equal(&mgmt_cfg, &new_cfg)) {
156 /* Reconnecting to the controller causes the config file to be
157 * resent automatically. If we're not reconnecting and the
158 * config file has changed, we need to notify the controller of
160 if (cfg_updated && mgmt_rconn) {
161 send_config_update(0, false);
163 svec_destroy(&new_cfg);
167 controller_name = cfg_get_string(0, "mgmt.controller");
168 if (!controller_name) {
169 VLOG_ERR("no controller specified for managment");
170 svec_destroy(&new_cfg);
174 max_backoff = cfg_get_int(0, "mgmt.max-backoff");
175 if (max_backoff < 1) {
176 max_backoff = MAX_BACKOFF_DEFAULT;
177 } else if (max_backoff > 3600) {
181 inactivity_probe = cfg_get_int(0, "mgmt.inactivity-probe");
182 if (inactivity_probe < 5) {
183 inactivity_probe = INACTIVITY_PROBE_DEFAULT;
186 /* xxx If this changes, we need to restart bridges to use new id,
187 * xxx but they need the id before the connect to controller, but we
188 * xxx need their dpids. */
189 /* Check if a different mgmt id has been assigned. */
190 if (cfg_has("mgmt.id")) {
191 uint64_t cfg_mgmt_id = cfg_get_dpid(0, "mgmt.id");
192 if (cfg_mgmt_id != mgmt_id) {
193 mgmt_id = cfg_mgmt_id;
197 svec_swap(&new_cfg, &mgmt_cfg);
198 svec_destroy(&new_cfg);
202 mgmt_configure_ssl();
206 rconn_destroy(mgmt_rconn);
209 mgmt_rconn = rconn_create(inactivity_probe, max_backoff);
210 retval = rconn_connect(mgmt_rconn, controller_name);
211 if (retval == EAFNOSUPPORT) {
212 VLOG_ERR("no support for %s vconn", controller_name);
217 make_ofmp_xid(size_t ofmp_len, uint16_t type, uint32_t xid,
218 struct ofpbuf **bufferp)
220 struct ofmp_header *oh;
222 oh = make_openflow_xid(ofmp_len, OFPT_VENDOR, xid, bufferp);
223 oh->header.vendor = htonl(NX_VENDOR_ID);
224 oh->header.subtype = htonl(NXT_MGMT);
225 oh->type = htons(type);
231 make_ofmp(size_t ofmp_len, uint16_t type, struct ofpbuf **bufferp)
233 struct ofmp_header *oh;
235 oh = make_openflow(ofmp_len, OFPT_VENDOR, bufferp);
236 oh->header.vendor = htonl(NX_VENDOR_ID);
237 oh->header.subtype = htonl(NXT_MGMT);
238 oh->type = htons(type);
244 send_openflow_buffer(struct ofpbuf *buffer)
249 VLOG_ERR("attempt to send openflow packet with no rconn\n");
253 /* OpenFlow messages use a 16-bit length field, so messages over 64K
254 * must be broken into multiple pieces.
256 if (buffer->size <= 65535) {
257 update_openflow_length(buffer);
258 retval = rconn_send_with_limit(mgmt_rconn, buffer, txqlen, TXQ_LIMIT);
260 VLOG_WARN_RL(&rl, "send to %s failed: %s",
261 rconn_get_name(mgmt_rconn), strerror(retval));
265 struct ofmp_header *header = (struct ofmp_header *)buffer->data;
266 uint32_t xid = header->header.header.xid;
267 size_t remain = buffer->size;
268 uint8_t *ptr = buffer->data;
270 /* Mark the OpenFlow header with a zero length to indicate some
273 header->header.header.length = 0;
276 struct ofpbuf *new_buffer;
277 struct ofmp_extended_data *oed;
278 size_t new_len = MIN(65535 - sizeof *oed, remain);
280 oed = make_ofmp_xid(sizeof *oed, OFMPT_EXTENDED_DATA, xid,
282 oed->type = header->type;
284 if (remain > 65535) {
285 oed->flags |= OFMPEDF_MORE_DATA;
288 printf("xxx SENDING LEN: %d\n", new_len);
290 /* Copy the entire original message, including the OpenFlow
291 * header, since management protocol structure definitions
292 * include these headers.
294 ofpbuf_put(new_buffer, ptr, new_len);
296 update_openflow_length(new_buffer);
297 retval = rconn_send_with_limit(mgmt_rconn, new_buffer, txqlen,
300 VLOG_WARN_RL(&rl, "send to %s failed: %s",
301 rconn_get_name(mgmt_rconn), strerror(retval));
302 ofpbuf_delete(buffer);
310 ofpbuf_delete(buffer);
316 send_features_reply(uint32_t xid)
318 struct ofpbuf *buffer;
319 struct ofp_switch_features *ofr;
321 ofr = make_openflow_xid(sizeof *ofr, OFPT_FEATURES_REPLY, xid, &buffer);
322 ofr->datapath_id = 0;
325 ofr->capabilities = 0;
327 send_openflow_buffer(buffer);
331 send_capability_reply(uint32_t xid)
334 struct ofpbuf *buffer;
335 struct ofmp_capability_reply *ofmpcr;
337 ofmpcr = make_ofmp_xid(sizeof *ofmpcr, OFMPT_CAPABILITY_REPLY,
339 ofmpcr->format = htonl(OFMPCOF_SIMPLE);
340 ofmpcr->mgmt_id = htonll(mgmt_id);
341 for (i=0; i<capabilities.n; i++) {
342 ofpbuf_put(buffer, capabilities.names[i],
343 strlen(capabilities.names[i]));
345 send_openflow_buffer(buffer);
349 send_resources_update(uint32_t xid, bool use_xid)
351 struct ofpbuf *buffer;
352 struct ofmp_resources_update *ofmpru;
353 struct ofmp_tlv *tlv;
355 struct svec port_list;
356 const char *host_uuid;
360 ofmpru = make_ofmp_xid(sizeof *ofmpru, OFMPT_RESOURCES_UPDATE,
363 ofmpru = make_ofmp(sizeof *ofmpru, OFMPT_RESOURCES_UPDATE, &buffer);
366 /* On XenServer systems, each host has its own UUID, which we provide
369 host_uuid = xenserver_get_host_uuid();
371 struct ofmptsr_mgmt_uuid *mgmt_uuid_tlv;
373 mgmt_uuid_tlv = ofpbuf_put_zeros(buffer, sizeof(*mgmt_uuid_tlv));
374 mgmt_uuid_tlv->type = htons(OFMPTSR_MGMT_UUID);
375 mgmt_uuid_tlv->len = htons(sizeof(*mgmt_uuid_tlv));
376 mgmt_uuid_tlv->mgmt_id = htonll(mgmt_id);
377 memcpy(mgmt_uuid_tlv->uuid, host_uuid, OFMP_UUID_LEN);
381 cfg_get_subsections(&br_list, "bridge");
382 for (i=0; i < br_list.n; i++) {
383 struct ofmptsr_dp *dp_tlv;
387 dp_id = bridge_get_datapathid(br_list.names[i]);
389 VLOG_WARN_RL(&rl, "bridge %s doesn't seem to exist",
393 dp_tlv = ofpbuf_put_zeros(buffer, sizeof(*dp_tlv));
394 dp_tlv->type = htons(OFMPTSR_DP);
395 dp_tlv->len = htons(sizeof(*dp_tlv));
397 dp_tlv->dp_id = htonll(dp_id);
398 memcpy(dp_tlv->name, br_list.names[i], strlen(br_list.names[i])+1);
400 /* On XenServer systems, each network has one or more UUIDs
401 * associated with it, which we provide to the controller.
403 n_uuid = cfg_count("bridge.%s.xs-network-uuids", br_list.names[i]);
405 struct ofmptsr_dp_uuid *dp_uuid_tlv;
406 size_t tlv_len = sizeof(*dp_uuid_tlv) + n_uuid * OFMP_UUID_LEN;
409 dp_uuid_tlv = ofpbuf_put_zeros(buffer, sizeof(*dp_uuid_tlv));
410 dp_uuid_tlv->type = htons(OFMPTSR_DP_UUID);
411 dp_uuid_tlv->len = htons(tlv_len);
412 dp_uuid_tlv->dp_id = htonll(dp_id);
414 for (j=0; j<n_uuid; j++) {
415 const char *dp_uuid = cfg_get_string(j,
416 "bridge.%s.xs-network-uuids", br_list.names[i]);
418 /* The UUID list could change underneath us, so just
419 * fill with zeros in that case. Another update will be
420 * initiated shortly, which should contain corrected data.
423 ofpbuf_put(buffer, dp_uuid, OFMP_UUID_LEN);
425 ofpbuf_put_zeros(buffer, OFMP_UUID_LEN);
430 svec_destroy(&br_list);
432 /* On XenServer systems, extended information about virtual interfaces
433 * (VIFs) is available, which is needed by the controller.
435 svec_init(&port_list);
436 bridge_get_ifaces(&port_list);
437 for (i=0; i < port_list.n; i++) {
438 const char *vif_uuid, *vm_uuid, *net_uuid;
440 struct ofmptsr_vif *vif_tlv;
442 vif_uuid = cfg_get_string(0, "port.%s.vif-uuid", port_list.names[i]);
447 vif_tlv = ofpbuf_put_zeros(buffer, sizeof(*vif_tlv));
448 vif_tlv->type = htons(OFMPTSR_VIF);
449 vif_tlv->len = htons(sizeof(*vif_tlv));
451 memcpy(vif_tlv->name, port_list.names[i], strlen(port_list.names[i])+1);
452 memcpy(vif_tlv->vif_uuid, vif_uuid, sizeof(vif_tlv->vif_uuid));
454 vm_uuid = cfg_get_string(0, "port.%s.vm-uuid", port_list.names[i]);
456 memcpy(vif_tlv->vm_uuid, vm_uuid, sizeof(vif_tlv->vm_uuid));
458 /* In case the vif disappeared underneath us. */
459 memset(vif_tlv->vm_uuid, '\0', sizeof(vif_tlv->vm_uuid));
462 net_uuid = cfg_get_string(0, "port.%s.net-uuid", port_list.names[i]);
464 memcpy(vif_tlv->net_uuid, net_uuid, sizeof(vif_tlv->net_uuid));
466 /* In case the vif disappeared underneath us. */
467 memset(vif_tlv->net_uuid, '\0', sizeof(vif_tlv->net_uuid));
470 vif_mac = cfg_get_mac(0, "port.%s.vif-mac", port_list.names[i]);
471 vif_tlv->vif_mac = htonll(vif_mac);
473 svec_destroy(&port_list);
475 /* Put end marker. */
476 tlv = ofpbuf_put_zeros(buffer, sizeof(*tlv));
477 tlv->type = htons(OFMPTSR_END);
478 tlv->len = htons(sizeof(*tlv));
479 send_openflow_buffer(buffer);
483 send_config_update(uint32_t xid, bool use_xid)
485 struct ofpbuf *buffer;
486 struct ofmp_config_update *ofmpcu;
489 ofmpcu = make_ofmp_xid(sizeof *ofmpcu, OFMPT_CONFIG_UPDATE,
492 ofmpcu = make_ofmp(sizeof *ofmpcu, OFMPT_CONFIG_UPDATE, &buffer);
495 ofmpcu->format = htonl(OFMPCOF_SIMPLE);
496 memcpy(ofmpcu->cookie, cfg_cookie, sizeof(ofmpcu->cookie));
498 send_openflow_buffer(buffer);
502 send_config_update_ack(uint32_t xid, bool success)
504 struct ofpbuf *buffer;
505 struct ofmp_config_update_ack *ofmpcua;
507 ofmpcua = make_ofmp_xid(sizeof *ofmpcua, OFMPT_CONFIG_UPDATE_ACK,
510 ofmpcua->format = htonl(OFMPCOF_SIMPLE);
512 ofmpcua->flags = htonl(OFMPCUAF_SUCCESS);
514 cfg_get_cookie(ofmpcua->cookie);
515 send_openflow_buffer(buffer);
519 send_ofmp_error_msg(uint32_t xid, uint16_t type, uint16_t code,
520 const void *data, size_t len)
522 struct ofpbuf *buffer;
523 struct ofmp_error_msg *oem;
525 oem = make_ofmp_xid(sizeof(*oem)+len, OFMPT_ERROR, xid, &buffer);
526 oem->type = htons(type);
527 oem->code = htons(code);
528 memcpy(oem->data, data, len);
529 send_openflow_buffer(buffer);
533 send_error_msg(uint32_t xid, uint16_t type, uint16_t code,
534 const void *data, size_t len)
536 struct ofpbuf *buffer;
537 struct ofp_error_msg *oem;
539 oem = make_openflow_xid(sizeof(*oem)+len, OFPT_ERROR, xid, &buffer);
540 oem->type = htons(type);
541 oem->code = htons(code);
542 memcpy(oem->data, data, len);
543 send_openflow_buffer(buffer);
547 recv_echo_request(uint32_t xid UNUSED, const void *msg)
549 const struct ofp_header *rq = msg;
550 send_openflow_buffer(make_echo_reply(rq));
555 recv_features_request(uint32_t xid, const void *msg UNUSED)
557 send_features_reply(xid);
562 recv_set_config(uint32_t xid UNUSED, const void *msg UNUSED)
564 /* Nothing to configure! */
569 recv_ofmp_capability_request(uint32_t xid, const struct ofmp_header *ofmph,
572 struct ofmp_capability_request *ofmpcr;
574 if (len != sizeof(*ofmpcr)) {
579 ofmpcr = (struct ofmp_capability_request *)ofmph;
580 if (ofmpcr->format != htonl(OFMPCAF_SIMPLE)) {
585 send_capability_reply(xid);
591 recv_ofmp_resources_request(uint32_t xid, const void *msg UNUSED,
594 send_resources_update(xid, true);
599 recv_ofmp_config_request(uint32_t xid, const struct ofmp_header *ofmph,
602 struct ofmp_config_request *ofmpcr;
604 if (len != sizeof(*ofmpcr)) {
609 ofmpcr = (struct ofmp_config_request *)ofmph;
610 if (ofmpcr->format != htonl(OFMPCOF_SIMPLE)) {
615 send_config_update(xid, true);
621 recv_ofmp_config_update(uint32_t xid, const struct ofmp_header *ofmph,
624 struct ofmp_config_update *ofmpcu;
627 data_len = len - sizeof(*ofmpcu);
628 if (data_len <= sizeof(*ofmpcu)) {
629 /* xxx Send error. */
633 ofmpcu = (struct ofmp_config_update *)ofmph;
634 if (ofmpcu->format != htonl(OFMPCOF_SIMPLE)) {
639 /* Check if the supplied cookie matches our current understanding of
640 * it. If they don't match, tell the controller and let it sort
642 if (cfg_lock(ofmpcu->cookie, 0)) {
643 /* xxx cfg_lock can fail for other reasons, such as being
645 VLOG_WARN_RL(&rl, "config update failed due to bad cookie\n");
646 send_config_update_ack(xid, false);
650 /* xxx We should probably do more sanity checking than this. */
652 cfg_write_data(ofmpcu->data, data_len);
655 /* Send the ACK before running reconfigure, since our management
656 * connection settings may have changed. */
657 send_config_update_ack(xid, true);
659 need_reconfigure = true;
665 recv_ofmp_extended_data(uint32_t xid, const struct ofmp_header *ofmph,
669 struct ofmp_extended_data *ofmped;
672 data_len = len - sizeof(*ofmped);
673 if (data_len <= sizeof(*ofmped)) {
674 /* xxx Send error. */
678 ofmped = (struct ofmp_extended_data *)ofmph;
680 ptr = ofpbuf_put(&ext_data_buffer, ofmped->data, data_len);
682 if (!ofmped->flags & OFMPEDF_MORE_DATA) {
683 recv_ofmp(xid, ext_data_buffer.data, ext_data_buffer.size);
684 ofpbuf_clear(&ext_data_buffer);
690 /* Handles receiving a management message. Generally, this function
691 * will be called 'len' set to zero, and the length will be derived by
692 * the OpenFlow header. With the extended data message, management
693 * messages are not constrained by OpenFlow's 64K message length limit.
694 * The extended data handler calls this function with the 'len' set to
695 * the total message length and the OpenFlow header's length field is
699 int recv_ofmp(uint32_t xid, struct ofmp_header *ofmph, size_t len)
702 len = ntohs(ofmph->header.header.length);
705 /* xxx Should sanity-check for min/max length */
706 switch (ntohs(ofmph->type))
708 case OFMPT_CAPABILITY_REQUEST:
709 return recv_ofmp_capability_request(xid, ofmph, len);
710 case OFMPT_RESOURCES_REQUEST:
711 return recv_ofmp_resources_request(xid, ofmph, len);
712 case OFMPT_CONFIG_REQUEST:
713 return recv_ofmp_config_request(xid, ofmph, len);
714 case OFMPT_CONFIG_UPDATE:
715 return recv_ofmp_config_update(xid, ofmph, len);
716 case OFMPT_EXTENDED_DATA:
717 return recv_ofmp_extended_data(xid, ofmph, len);
719 VLOG_WARN_RL(&rl, "unknown mgmt message: %d",
726 recv_nx_msg(uint32_t xid, const void *oh)
728 const struct nicira_header *nh = oh;
730 switch (ntohl(nh->subtype)) {
733 return recv_ofmp(xid, (struct ofmp_header *)oh, 0);
736 send_error_msg(xid, OFPET_BAD_REQUEST, OFPBRC_BAD_SUBTYPE,
737 oh, ntohs(nh->header.length));
743 recv_vendor(uint32_t xid, const void *oh)
745 const struct ofp_vendor_header *ovh = oh;
747 switch (ntohl(ovh->vendor))
750 return recv_nx_msg(xid, oh);
753 VLOG_WARN_RL(&rl, "unknown vendor: 0x%x", ntohl(ovh->vendor));
754 send_error_msg(xid, OFPET_BAD_REQUEST, OFPBRC_BAD_VENDOR,
755 oh, ntohs(ovh->header.length));
761 handle_msg(uint32_t xid, const void *msg, size_t length)
763 int (*handler)(uint32_t, const void *);
764 struct ofp_header *oh;
767 COVERAGE_INC(mgmt_received);
769 /* Check encapsulated length. */
770 oh = (struct ofp_header *) msg;
771 if (ntohs(oh->length) > length) {
774 assert(oh->version == OFP_VERSION);
776 /* Figure out how to handle it. */
778 case OFPT_ECHO_REQUEST:
779 min_size = sizeof(struct ofp_header);
780 handler = recv_echo_request;
782 case OFPT_ECHO_REPLY:
784 case OFPT_FEATURES_REQUEST:
785 min_size = sizeof(struct ofp_header);
786 handler = recv_features_request;
788 case OFPT_SET_CONFIG:
789 min_size = sizeof(struct ofp_switch_config);
790 handler = recv_set_config;
793 min_size = sizeof(struct ofp_vendor_header);
794 handler = recv_vendor;
797 VLOG_WARN_RL(&rl, "unknown openflow type: %d", oh->type);
798 send_error_msg(xid, OFPET_BAD_REQUEST, OFPBRC_BAD_TYPE,
804 if (length < min_size) {
807 return handler(xid, msg);
819 need_reconfigure = false;
820 rconn_run(mgmt_rconn);
822 /* Do some processing, but cap it at a reasonable amount so that
823 * other processing doesn't starve. */
824 for (i=0; i<50; i++) {
825 struct ofpbuf *buffer;
826 struct ofp_header *oh;
828 buffer = rconn_recv(mgmt_rconn);
833 if (buffer->size >= sizeof *oh) {
835 handle_msg(oh->xid, buffer->data, buffer->size);
836 ofpbuf_delete(buffer);
838 VLOG_WARN_RL(&rl, "received too-short OpenFlow message");
842 return need_reconfigure;
852 rconn_run_wait(mgmt_rconn);
853 rconn_recv_wait(mgmt_rconn);
857 pick_fallback_mgmt_id(void)
859 uint8_t ea[ETH_ADDR_LEN];
861 ea[0] = 0x00; /* Set Nicira OUI. */
864 return eth_addr_to_uint64(ea);