From b99d3ceeedd2c5533c13f9794616fc6d9c49c0de Mon Sep 17 00:00:00 2001 From: Ben Pfaff Date: Tue, 17 Apr 2012 21:52:10 -0700 Subject: [PATCH] ofproto-dpif: Batch flow uninstallations due to expiration. Signed-off-by: Ben Pfaff --- lib/dpif-linux.c | 103 ++++++++++++++++++++++++++++------------- lib/dpif-netdev.c | 10 ++-- lib/dpif-provider.h | 14 +++--- lib/dpif.c | 52 ++++++++++++++++----- lib/dpif.h | 13 +++++- ofproto/ofproto-dpif.c | 50 +++++++++++++++++++- 6 files changed, 182 insertions(+), 60 deletions(-) diff --git a/lib/dpif-linux.c b/lib/dpif-linux.c index 7013bc54..d47ddf15 100644 --- a/lib/dpif-linux.c +++ b/lib/dpif-linux.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2009, 2010, 2011 Nicira Networks. + * Copyright (c) 2008, 2009, 2010, 2011, 2012 Nicira Networks. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -645,26 +645,32 @@ dpif_linux_flow_put(struct dpif *dpif_, const struct dpif_flow_put *put) return error; } -static int -dpif_linux_flow_del(struct dpif *dpif_, - const struct nlattr *key, size_t key_len, - struct dpif_flow_stats *stats) +static void +dpif_linux_init_flow_del(struct dpif *dpif_, const struct dpif_flow_del *del, + struct dpif_linux_flow *request) { struct dpif_linux *dpif = dpif_linux_cast(dpif_); + + dpif_linux_flow_init(request); + request->cmd = OVS_FLOW_CMD_DEL; + request->dp_ifindex = dpif->dp_ifindex; + request->key = del->key; + request->key_len = del->key_len; +} + +static int +dpif_linux_flow_del(struct dpif *dpif_, const struct dpif_flow_del *del) +{ struct dpif_linux_flow request, reply; struct ofpbuf *buf; int error; - dpif_linux_flow_init(&request); - request.cmd = OVS_FLOW_CMD_DEL; - request.dp_ifindex = dpif->dp_ifindex; - request.key = key; - request.key_len = key_len; + dpif_linux_init_flow_del(dpif_, del, &request); error = dpif_linux_flow_transact(&request, - stats ? &reply : NULL, - stats ? &buf : NULL); - if (!error && stats) { - dpif_linux_flow_get_stats(&reply, stats); + del->stats ? &reply : NULL, + del->stats ? &buf : NULL); + if (!error && del->stats) { + dpif_linux_flow_get_stats(&reply, del->stats); ofpbuf_delete(buf); } return error; @@ -818,23 +824,39 @@ dpif_linux_operate(struct dpif *dpif_, struct dpif_op **ops, size_t n_ops) for (i = 0; i < n_ops; i++) { struct nl_transaction *txn = &txns[i]; struct dpif_op *op = ops[i]; - - if (op->type == DPIF_OP_FLOW_PUT) { - struct dpif_flow_put *put = &op->u.flow_put; - struct dpif_linux_flow request; - + struct dpif_flow_put *put; + struct dpif_flow_del *del; + struct dpif_execute *execute; + struct dpif_linux_flow request; + + switch (op->type) { + case DPIF_OP_FLOW_PUT: + put = &op->u.flow_put; dpif_linux_init_flow_put(dpif_, put, &request); if (put->stats) { request.nlmsg_flags |= NLM_F_ECHO; } txn->request = ofpbuf_new(1024); dpif_linux_flow_to_ofpbuf(&request, txn->request); - } else if (op->type == DPIF_OP_EXECUTE) { - struct dpif_execute *execute = &op->u.execute; + break; + + case DPIF_OP_FLOW_DEL: + del = &op->u.flow_del; + dpif_linux_init_flow_del(dpif_, del, &request); + if (del->stats) { + request.nlmsg_flags |= NLM_F_ECHO; + } + txn->request = ofpbuf_new(1024); + dpif_linux_flow_to_ofpbuf(&request, txn->request); + break; + case DPIF_OP_EXECUTE: + execute = &op->u.execute; txn->request = dpif_linux_encode_execute(dpif->dp_ifindex, execute); - } else { + break; + + default: NOT_REACHED(); } } @@ -851,23 +873,40 @@ dpif_linux_operate(struct dpif *dpif_, struct dpif_op **ops, size_t n_ops) for (i = 0; i < n_ops; i++) { struct nl_transaction *txn = &txns[i]; struct dpif_op *op = ops[i]; + struct dpif_flow_put *put; + struct dpif_flow_del *del; - if (op->type == DPIF_OP_FLOW_PUT) { - struct dpif_flow_put *put = &op->u.flow_put; - int error = txn->error; + op->error = txn->error; - if (!error && put->stats) { + switch (op->type) { + case DPIF_OP_FLOW_PUT: + put = &op->u.flow_put; + if (!op->error && put->stats) { struct dpif_linux_flow reply; - error = dpif_linux_flow_from_ofpbuf(&reply, txn->reply); - if (!error) { + op->error = dpif_linux_flow_from_ofpbuf(&reply, txn->reply); + if (!op->error) { dpif_linux_flow_get_stats(&reply, put->stats); } } - op->error = error; - } else if (op->type == DPIF_OP_EXECUTE) { - op->error = txn->error; - } else { + break; + + case DPIF_OP_FLOW_DEL: + del = &op->u.flow_del; + if (!op->error && del->stats) { + struct dpif_linux_flow reply; + + op->error = dpif_linux_flow_from_ofpbuf(&reply, txn->reply); + if (!op->error) { + dpif_linux_flow_get_stats(&reply, del->stats); + } + } + break; + + case DPIF_OP_EXECUTE: + break; + + default: NOT_REACHED(); } diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c index 16860870..a9077227 100644 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -777,24 +777,22 @@ dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put) } static int -dpif_netdev_flow_del(struct dpif *dpif, - const struct nlattr *nl_key, size_t nl_key_len, - struct dpif_flow_stats *stats) +dpif_netdev_flow_del(struct dpif *dpif, const struct dpif_flow_del *del) { struct dp_netdev *dp = get_dp_netdev(dpif); struct dp_netdev_flow *flow; struct flow key; int error; - error = dpif_netdev_flow_from_nlattrs(nl_key, nl_key_len, &key); + error = dpif_netdev_flow_from_nlattrs(del->key, del->key_len, &key); if (error) { return error; } flow = dp_netdev_lookup_flow(dp, &key); if (flow) { - if (stats) { - get_dpif_flow_stats(flow, stats); + if (del->stats) { + get_dpif_flow_stats(flow, del->stats); } dp_netdev_free_flow(dp, flow); return 0; diff --git a/lib/dpif-provider.h b/lib/dpif-provider.h index 5ef4c991..75c65bba 100644 --- a/lib/dpif-provider.h +++ b/lib/dpif-provider.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2010, 2011 Nicira Networks. + * Copyright (c) 2009, 2010, 2011, 2012 Nicira Networks. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -235,14 +235,12 @@ struct dpif_class { /* Deletes a flow from 'dpif' and returns 0, or returns ENOENT if 'dpif' * does not contain such a flow. The flow is specified by the Netlink - * attributes with types OVS_KEY_ATTR_* in the 'key_len' bytes starting at - * 'key'. + * attributes with types OVS_KEY_ATTR_* in the 'del->key_len' bytes + * starting at 'del->key'. * - * If the operation succeeds, then 'stats', if nonnull, must be set to the - * flow's statistics before its deletion. */ - int (*flow_del)(struct dpif *dpif, - const struct nlattr *key, size_t key_len, - struct dpif_flow_stats *stats); + * If the operation succeeds, then 'del->stats', if nonnull, must be set to + * the flow's statistics before its deletion. */ + int (*flow_del)(struct dpif *dpif, const struct dpif_flow_del *del); /* Deletes all flows from 'dpif' and clears all of its queues of received * packets. */ diff --git a/lib/dpif.c b/lib/dpif.c index 73696e4d..35df1e5d 100644 --- a/lib/dpif.c +++ b/lib/dpif.c @@ -89,6 +89,8 @@ static void log_operation(const struct dpif *, const char *operation, static bool should_log_flow_message(int error); static void log_flow_put_message(struct dpif *, const struct dpif_flow_put *, int error); +static void log_flow_del_message(struct dpif *, const struct dpif_flow_del *, + int error); static void log_execute_message(struct dpif *, const struct dpif_execute *, int error); @@ -815,6 +817,21 @@ dpif_flow_put(struct dpif *dpif, enum dpif_flow_put_flags flags, return dpif_flow_put__(dpif, &put); } +static int +dpif_flow_del__(struct dpif *dpif, struct dpif_flow_del *del) +{ + int error; + + COVERAGE_INC(dpif_flow_del); + + error = dpif->dpif_class->flow_del(dpif, del); + if (error && del->stats) { + memset(del->stats, 0, sizeof *del->stats); + } + log_flow_del_message(dpif, del, error); + return error; +} + /* Deletes a flow from 'dpif' and returns 0, or returns ENOENT if 'dpif' does * not contain such a flow. The flow is specified by the Netlink attributes * with types OVS_KEY_ATTR_* in the 'key_len' bytes starting at 'key'. @@ -826,19 +843,12 @@ dpif_flow_del(struct dpif *dpif, const struct nlattr *key, size_t key_len, struct dpif_flow_stats *stats) { - int error; + struct dpif_flow_del del; - COVERAGE_INC(dpif_flow_del); - - error = dpif->dpif_class->flow_del(dpif, key, key_len, stats); - if (error && stats) { - memset(stats, 0, sizeof *stats); - } - if (should_log_flow_message(error)) { - log_flow_message(dpif, error, "flow_del", key, key_len, - !error ? stats : NULL, NULL, 0); - } - return error; + del.key = key; + del.key_len = key_len; + del.stats = stats; + return dpif_flow_del__(dpif, &del); } /* Initializes 'dump' to begin dumping the flows in a dpif. @@ -994,6 +1004,10 @@ dpif_operate(struct dpif *dpif, struct dpif_op **ops, size_t n_ops) log_flow_put_message(dpif, &op->u.flow_put, op->error); break; + case DPIF_OP_FLOW_DEL: + log_flow_del_message(dpif, &op->u.flow_del, op->error); + break; + case DPIF_OP_EXECUTE: log_execute_message(dpif, &op->u.execute, op->error); break; @@ -1010,6 +1024,10 @@ dpif_operate(struct dpif *dpif, struct dpif_op **ops, size_t n_ops) op->error = dpif_flow_put__(dpif, &op->u.flow_put); break; + case DPIF_OP_FLOW_DEL: + op->error = dpif_flow_del__(dpif, &op->u.flow_del); + break; + case DPIF_OP_EXECUTE: op->error = dpif_execute__(dpif, &op->u.execute); break; @@ -1245,6 +1263,16 @@ log_flow_put_message(struct dpif *dpif, const struct dpif_flow_put *put, } } +static void +log_flow_del_message(struct dpif *dpif, const struct dpif_flow_del *del, + int error) +{ + if (should_log_flow_message(error)) { + log_flow_message(dpif, error, "flow_del", del->key, del->key_len, + !error ? del->stats : NULL, NULL, 0); + } +} + static void log_execute_message(struct dpif *dpif, const struct dpif_execute *execute, int error) diff --git a/lib/dpif.h b/lib/dpif.h index 1a6ca053..768934b1 100644 --- a/lib/dpif.h +++ b/lib/dpif.h @@ -177,7 +177,8 @@ int dpif_execute(struct dpif *, enum dpif_op_type { DPIF_OP_FLOW_PUT = 1, - DPIF_OP_EXECUTE + DPIF_OP_FLOW_DEL, + DPIF_OP_EXECUTE, }; struct dpif_flow_put { @@ -192,6 +193,15 @@ struct dpif_flow_put { struct dpif_flow_stats *stats; /* Optional flow statistics. */ }; +struct dpif_flow_del { + /* Input. */ + const struct nlattr *key; /* Flow to delete. */ + size_t key_len; /* Length of 'key' in bytes. */ + + /* Output. */ + struct dpif_flow_stats *stats; /* Optional flow statistics. */ +}; + struct dpif_execute { const struct nlattr *key; /* Partial flow key (only for metadata). */ size_t key_len; /* Length of 'key' in bytes. */ @@ -205,6 +215,7 @@ struct dpif_op { int error; union { struct dpif_flow_put flow_put; + struct dpif_flow_del flow_del; struct dpif_execute execute; } u; }; diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c index f9936e4c..9660a1f8 100644 --- a/ofproto/ofproto-dpif.c +++ b/ofproto/ofproto-dpif.c @@ -2823,6 +2823,9 @@ handle_miss_upcalls(struct ofproto_dpif *ofproto, struct dpif_upcall *upcalls, op->subfacet->installed = true; } break; + + case DPIF_OP_FLOW_DEL: + NOT_REACHED(); } } HMAP_FOR_EACH_SAFE (miss, next_miss, hmap_node, &todo) { @@ -3112,18 +3115,63 @@ subfacet_max_idle(const struct ofproto_dpif *ofproto) return bucket * BUCKET_WIDTH; } +enum { EXPIRE_MAX_BATCH = 50 }; + +static void +expire_batch(struct ofproto_dpif *ofproto, struct subfacet **subfacets, int n) +{ + struct odputil_keybuf keybufs[EXPIRE_MAX_BATCH]; + struct dpif_op ops[EXPIRE_MAX_BATCH]; + struct dpif_op *opsp[EXPIRE_MAX_BATCH]; + struct ofpbuf keys[EXPIRE_MAX_BATCH]; + struct dpif_flow_stats stats[EXPIRE_MAX_BATCH]; + int i; + + for (i = 0; i < n; i++) { + ops[i].type = DPIF_OP_FLOW_DEL; + subfacet_get_key(subfacets[i], &keybufs[i], &keys[i]); + ops[i].u.flow_del.key = keys[i].data; + ops[i].u.flow_del.key_len = keys[i].size; + ops[i].u.flow_del.stats = &stats[i]; + opsp[i] = &ops[i]; + } + + dpif_operate(ofproto->dpif, opsp, n); + for (i = 0; i < n; i++) { + subfacet_reset_dp_stats(subfacets[i], &stats[i]); + subfacets[i]->installed = false; + subfacet_destroy(subfacets[i]); + } +} + static void expire_subfacets(struct ofproto_dpif *ofproto, int dp_max_idle) { long long int cutoff = time_msec() - dp_max_idle; + struct subfacet *subfacet, *next_subfacet; + struct subfacet *batch[EXPIRE_MAX_BATCH]; + int n_batch; + n_batch = 0; HMAP_FOR_EACH_SAFE (subfacet, next_subfacet, hmap_node, &ofproto->subfacets) { if (subfacet->used < cutoff) { - subfacet_destroy(subfacet); + if (subfacet->installed) { + batch[n_batch++] = subfacet; + if (n_batch >= EXPIRE_MAX_BATCH) { + expire_batch(ofproto, batch, n_batch); + n_batch = 0; + } + } else { + subfacet_destroy(subfacet); + } } } + + if (n_batch > 0) { + expire_batch(ofproto, batch, n_batch); + } } /* If 'rule' is an OpenFlow rule, that has expired according to OpenFlow rules, -- 2.30.2