secchan: Implement OFPP_TABLE and NXAST_RESUBMIT actions.
authorBen Pfaff <blp@nicira.com>
Tue, 3 Mar 2009 00:52:55 +0000 (16:52 -0800)
committerBen Pfaff <blp@nicira.com>
Tue, 3 Mar 2009 00:52:55 +0000 (16:52 -0800)
MISSING
include/openflow/nicira-ext.h
lib/vconn.c
secchan/ofproto.c

diff --git a/MISSING b/MISSING
index ffe5160ac221352d4bef275c11a87f9d43ca6cf9..6da63755eaa83fe673ebde36d50e64e0b6ae7df2 100644 (file)
--- a/MISSING
+++ b/MISSING
@@ -9,13 +9,9 @@ reimplement them with the new architecture:
 - The OFPPC_NO_RECV, OFPPC_NO_RECV_STP, and OFPPC_NO_FWD bits in port
   configurations.
 
-- The OFPP_TABLE action.
-
 - SNAT support in secchan (but SNAT is still supported in the kernel
   datapath).
 
 - udatapath.
 
-- vswitchd (this is our top priority).
-
 - A lot of the manpages and documentation need to be updated.
index a836ba7c14e38d36a048ab10982687d430d9a0b0..0d07584c69b15949fe14ce1d5c171702816cfd92 100644 (file)
@@ -101,7 +101,8 @@ OFP_ASSERT(sizeof(struct nx_act_config) == 20);
 
 
 enum nx_action_subtype {
-    NXAST_SNAT                      /* Source-NAT */
+    NXAST_SNAT,                     /* Source-NAT */
+    NXAST_RESUBMIT                  /* Throw against flow table again. */
 };
 
 /* Action structure for NXAST_SNAT. */
@@ -116,6 +117,17 @@ struct nx_action_snat {
 };
 OFP_ASSERT(sizeof(struct nx_action_snat) == 16);
 
+/* Action structure for NXAST_RESUBMIT. */
+struct nx_action_resubmit {
+    uint16_t type;                  /* OFPAT_VENDOR. */
+    uint16_t len;                   /* Length is 8. */
+    uint32_t vendor;                /* NX_VENDOR_ID. */
+    uint16_t subtype;               /* NXAST_RESUBMIT. */
+    uint16_t in_port;               /* New in_port for checking flow table. */
+    uint8_t pad[4];
+};
+OFP_ASSERT(sizeof(struct nx_action_resubmit) == 16);
+
 /* Header for Nicira-defined actions. */
 struct nx_action_header {
     uint16_t type;                  /* OFPAT_VENDOR. */
index bd8e255feadb3fbca02803a7a08bbaf4ad83a8b4..6755734a9a07b79335e122e9baaee1129909c7e9 100644 (file)
@@ -1204,17 +1204,14 @@ check_action_port(int port)
 {
     switch (port) {
     case OFPP_IN_PORT:
+    case OFPP_TABLE:
+    case OFPP_NORMAL:
     case OFPP_FLOOD:
     case OFPP_ALL:
     case OFPP_CONTROLLER:
     case OFPP_LOCAL:
         return 0;
 
-    case OFPP_TABLE:
-    case OFPP_NORMAL:
-        VLOG_WARN_RL(&bad_ofmsg_rl, "port %x not yet implemented", port);
-        return ofp_mkerr(OFPET_BAD_ACTION, OFPBAC_BAD_OUT_PORT);
-
     default:
         if (port >= 0 && port < OFPP_MAX) {
             return 0;
@@ -1244,6 +1241,8 @@ check_nicira_action(const union ofp_action *a, unsigned int len)
             return error;
         }
         return check_action_port(ntohs(((struct nx_action_snat *) nah)->port));
+    case NXAST_RESUBMIT:
+        return check_action_exact_len(a, len, 16);
     default:
         return ofp_mkerr(OFPET_BAD_ACTION, OFPBAC_BAD_VENDOR_TYPE);
     }
index 2abb59525cf00f8043c26947efa1bcb29312ee48..00a1616736942d016e3b5245f2323d85fb1bd6b1 100644 (file)
@@ -91,9 +91,9 @@ struct odp_actions {
 
 static void init_actions(struct odp_actions *);
 static void free_actions(struct odp_actions *);
-static void ofp_actions_to_odp_actions(uint16_t ofp_in_port,
-                                       const struct ofp_action_header *in_,
-                                       size_t n_in, struct odp_actions *out);
+static void xlate_actions(const union ofp_action *in, size_t n_in,
+                          const flow_t *flow, struct ofproto *ofproto,
+                          struct odp_actions *out);
 
 #define UNKNOWN_SUPER ((struct rule *)-1)
 struct rule {
@@ -119,7 +119,8 @@ struct rule {
 static void rule_destroy(struct rule *);
 static inline size_t rule_size(int n_actions);
 static struct rule *rule_from_cls_rule(const struct cls_rule *);
-static void rule_make_actions(const struct rule *, struct odp_actions *);
+static void rule_make_actions(struct ofproto *,
+                              const struct rule *, struct odp_actions *);
 
 struct ofconn {
     struct list node;
@@ -177,7 +178,8 @@ static void send_packet_in_miss(struct ofpbuf *, void *ofproto);
 static void send_packet_in_action(struct ofpbuf *, void *ofproto);
 static void update_used(struct ofproto *);
 static void expire_rule(struct cls_rule *, void *ofproto);
-static void revalidate_subrule(struct cls_rule *, void *ofproto);
+static bool revalidate_subrule(struct ofproto *p, struct rule *subrule);
+static void revalidate_subrule_cb(struct cls_rule *sub_, void *p_);
 
 static void handle_odp_msg(struct ofproto *, struct ofpbuf *);
 
@@ -390,7 +392,8 @@ ofproto_run(struct ofproto *p)
     }
 
     if (p->need_revalidate) {
-        classifier_for_each_with_wildcards(&p->cls, 0, revalidate_subrule, p);
+        classifier_for_each_with_wildcards(&p->cls, 0, revalidate_subrule_cb,
+                                           p);
         p->need_revalidate = false;
     }
 }
@@ -446,9 +449,7 @@ ofproto_send_packet(struct ofproto *p, const flow_t *flow,
     struct odp_actions odp_actions;
     int error;
 
-    ofp_actions_to_odp_actions(odp_port_to_ofp_port(flow->in_port),
-                               (const struct ofp_action_header *) actions,
-                               n_actions, &odp_actions);
+    xlate_actions(actions, n_actions, flow, p, &odp_actions);
     error = dpif_execute(&p->dpif, flow->in_port, odp_actions.actions,
                          odp_actions.n_actions, packet);
     free_actions(&odp_actions);
@@ -484,7 +485,7 @@ ofproto_setup_exact_flow(struct ofproto *p, const flow_t *flow,
         rule_destroy(displaced_rule);
     }
 
-    rule_make_actions(rule, &odp_actions);
+    rule_make_actions(p, rule, &odp_actions);
     if (packet) {
         if (!ofproto_send_packet(p, flow, actions, n_actions, packet)) {
             rule->byte_count = packet->size;
@@ -845,13 +846,13 @@ rule_has_out_port(const struct rule *rule, uint16_t out_port)
 }
 
 static void
-rule_make_actions(const struct rule *rule, struct odp_actions *actions)
+rule_make_actions(struct ofproto *p,
+                  const struct rule *rule, struct odp_actions *actions)
 {
     const struct rule *super = rule->super ? rule->super : rule;
     assert(!rule->cr.wc.wildcards);
-    ofp_actions_to_odp_actions(odp_port_to_ofp_port(rule->cr.flow.in_port),
-                               (const struct ofp_action_header *) super->actions,
-                               super->n_actions, actions);
+    xlate_actions(super->actions, super->n_actions, &rule->cr.flow, p,
+                  actions);
 }
 \f
 static void
@@ -1079,105 +1080,180 @@ add_controller_action(struct odp_actions *actions,
     a->controller.arg = oao->max_len ? ntohs(oao->max_len) : UINT32_MAX;
 }
 
-static int
-ofp_to_odp_action_output(struct odp_actions *actions, uint16_t ofp_in_port,
-                         const struct ofp_action_output *oao)
+struct action_xlate_ctx {
+    /* Input. */
+    const union ofp_action *in; /* OpenFlow actions. */
+    size_t n_in;                /* Number of elements in 'in' array. */
+    const flow_t *flow;         /* Flow to which these actions correspond. */
+    struct ofproto *ofproto;    /* For OFPP_TABLE, NXAST_RESUBMIT only. */
+
+    /* Output. */
+    struct odp_actions *out;    /* Datapath actions. */
+};
+
+static void do_xlate_actions(struct action_xlate_ctx *ctx);
+
+static void
+xlate_table_action(const struct action_xlate_ctx *ctx, uint16_t in_port)
+{
+    struct ofproto *p = ctx->ofproto;
+    struct action_xlate_ctx nested_ctx;
+    struct rule *rule;
+    flow_t flow;
+
+    if (!p) {
+        return;
+    }
+
+    flow = *ctx->flow;
+    flow.in_port = in_port;
+
+    rule = rule_from_cls_rule(classifier_lookup(&p->cls, &flow));
+    if (!rule) {
+        return;
+    } else if (rule->super && p->need_revalidate) {
+        /* This might be a subrule that is now invalid.  Revalidate it. */
+        if (!revalidate_subrule(p, rule)) {
+            /* The subrule got deleted so we can optimize slightly by only
+             * look through the wildcarded rules. */
+            rule = rule_from_cls_rule(classifier_lookup_wild(&p->cls, &flow));
+            if (!rule) {
+                return;
+            }
+        }
+    }
+    if (rule->super) {
+        rule = rule->super;
+    }
+
+    nested_ctx.in = rule->actions;
+    nested_ctx.n_in = rule->n_actions;
+    nested_ctx.flow = &flow;
+    nested_ctx.ofproto = NULL; /* Prevent recursion. */
+    nested_ctx.out = ctx->out;
+    do_xlate_actions(&nested_ctx);
+}
+
+static void
+xlate_output_action(const struct action_xlate_ctx *ctx,
+                    const struct ofp_action_output *oao)
 {
+    uint16_t odp_port;
+
     switch (ntohs(oao->port)) {
     case OFPP_IN_PORT:
-        add_output_action(actions, ofp_port_to_odp_port(ofp_in_port));
+        add_output_action(ctx->out, ctx->flow->in_port);
         break;
     case OFPP_TABLE:
-        /* XXX not implemented */
+        xlate_table_action(ctx, ctx->flow->in_port);
         break;
     case OFPP_NORMAL:
-        add_output_group_action(actions, DP_GROUP_FLOOD); /* XXX */
+        add_output_group_action(ctx->out, DP_GROUP_FLOOD); /* XXX */
         break;
     case OFPP_FLOOD:
-        add_output_group_action(actions, DP_GROUP_FLOOD);
+        add_output_group_action(ctx->out, DP_GROUP_FLOOD);
         break;
     case OFPP_ALL:
-        add_output_group_action(actions, DP_GROUP_ALL);
+        add_output_group_action(ctx->out, DP_GROUP_ALL);
         break;
     case OFPP_CONTROLLER:
-        add_controller_action(actions, oao);
+        add_controller_action(ctx->out, oao);
         break;
     case OFPP_LOCAL:
-        add_output_action(actions, ODPP_LOCAL);
+        add_output_action(ctx->out, ODPP_LOCAL);
         break;
     default:
-        if (ntohs(oao->port) != ofp_in_port) {
-            add_output_action(actions, ofp_port_to_odp_port(ntohs(oao->port)));
+        odp_port = ofp_port_to_odp_port(ntohs(oao->port));
+        if (odp_port != ctx->flow->in_port) {
+            add_output_action(ctx->out, odp_port);
         }
         break;
     }
-    return 0;
 }
 
 static void
-ofp_actions_to_odp_actions(uint16_t ofp_in_port,
-                           const struct ofp_action_header *in_, size_t n_in,
-                           struct odp_actions *out)
+xlate_nicira_action(const struct action_xlate_ctx *ctx,
+                    const struct nx_action_header *nah)
+{
+    const struct nx_action_snat *nas;
+    const struct nx_action_resubmit *nar;
+    int subtype = ntohs(nah->subtype);
+    union odp_action *oa;
+
+    assert(nah->vendor == htonl(NX_VENDOR_ID));
+    switch (subtype) {
+    case NXAST_SNAT:
+        nas = (const struct nx_action_snat *) nah;
+        oa = add_action(ctx->out, ODPAT_SNAT);
+        oa->snat.port = ntohs(nas->port);
+        break;
+
+    case NXAST_RESUBMIT:
+        nar = (const struct nx_action_resubmit *) nah;
+        xlate_table_action(ctx, ofp_port_to_odp_port(ntohs(nar->in_port)));
+        break;
+
+    default:
+        VLOG_DBG_RL(&rl, "unknown Nicira action type %"PRIu16, subtype);
+        break;
+    }
+}
+
+static void
+do_xlate_actions(struct action_xlate_ctx *ctx)
 {
-    union ofp_action *in = (union ofp_action *) in_;
     struct actions_iterator iter;
-    const union ofp_action *a;
+    const union ofp_action *ia;
 
-    init_actions(out);
-    for (a = actions_first(&iter, in, n_in); a; a = actions_next(&iter)) {
-        uint16_t type = ntohs(a->type);
+    for (ia = actions_first(&iter, ctx->in, ctx->n_in); ia;
+         ia = actions_next(&iter))
+    {
+        uint16_t type = ntohs(ia->type);
         union odp_action *oa;
 
         switch (type) {
         case OFPAT_OUTPUT:
-            ofp_to_odp_action_output(out, ofp_in_port, &a->output);
+            xlate_output_action(ctx, &ia->output);
             break;
 
         case OFPAT_SET_VLAN_VID:
-            oa = add_action(out, ODPAT_SET_VLAN_VID);
-            oa->vlan_vid.vlan_vid = a->vlan_vid.vlan_vid;
+            oa = add_action(ctx->out, ODPAT_SET_VLAN_VID);
+            oa->vlan_vid.vlan_vid = ia->vlan_vid.vlan_vid;
             break;
 
         case OFPAT_SET_VLAN_PCP:
-            oa = add_action(out, ODPAT_SET_VLAN_PCP);
-            oa->vlan_pcp.vlan_pcp = a->vlan_pcp.vlan_pcp;
+            oa = add_action(ctx->out, ODPAT_SET_VLAN_PCP);
+            oa->vlan_pcp.vlan_pcp = ia->vlan_pcp.vlan_pcp;
             break;
 
         case OFPAT_STRIP_VLAN:
-            add_action(out, ODPAT_STRIP_VLAN);
+            add_action(ctx->out, ODPAT_STRIP_VLAN);
             break;
 
         case OFPAT_SET_DL_SRC:
-            oa = add_action(out, ODPAT_SET_DL_SRC);
+            oa = add_action(ctx->out, ODPAT_SET_DL_SRC);
             memcpy(oa->dl_addr.dl_addr,
-                   ((struct ofp_action_dl_addr *) a)->dl_addr, ETH_ADDR_LEN);
+                   ((struct ofp_action_dl_addr *) ia)->dl_addr, ETH_ADDR_LEN);
             break;
 
         case OFPAT_SET_DL_DST:
-            oa = add_action(out, ODPAT_SET_DL_DST);
+            oa = add_action(ctx->out, ODPAT_SET_DL_DST);
             memcpy(oa->dl_addr.dl_addr,
-                   ((struct ofp_action_dl_addr *) a)->dl_addr, ETH_ADDR_LEN);
+                   ((struct ofp_action_dl_addr *) ia)->dl_addr, ETH_ADDR_LEN);
             break;
 
         case OFPAT_SET_NW_SRC:
-            oa = add_action(out, ODPAT_SET_NW_SRC);
-            oa->nw_addr.nw_addr = a->nw_addr.nw_addr;
+            oa = add_action(ctx->out, ODPAT_SET_NW_SRC);
+            oa->nw_addr.nw_addr = ia->nw_addr.nw_addr;
             break;
 
         case OFPAT_SET_TP_SRC:
-            oa = add_action(out, ODPAT_SET_TP_SRC);
-            oa->tp_port.tp_port = a->tp_port.tp_port;
+            oa = add_action(ctx->out, ODPAT_SET_TP_SRC);
+            oa->tp_port.tp_port = ia->tp_port.tp_port;
             break;
 
         case OFPAT_VENDOR:
-            if (a->vendor.vendor == htonl(NX_VENDOR_ID)) {
-                const struct nx_action_snat *nas =
-                    (const struct nx_action_snat *) a;
-                if (nas->subtype == htons(NXAST_SNAT)) {
-                    oa = add_action(out, ODPAT_SNAT);
-                    oa->snat.port = ntohs(nas->port);
-                }
-            }
+            xlate_nicira_action(ctx, (const struct nx_action_header *) ia);
             break;
 
         default:
@@ -1187,6 +1263,21 @@ ofp_actions_to_odp_actions(uint16_t ofp_in_port,
     }
 }
 
+static void
+xlate_actions(const union ofp_action *in, size_t n_in,
+              const flow_t *flow, struct ofproto *ofproto,
+              struct odp_actions *out)
+{
+    struct action_xlate_ctx ctx;
+    init_actions(out);
+    ctx.in = in;
+    ctx.n_in = n_in;
+    ctx.flow = flow;
+    ctx.ofproto = ofproto;
+    ctx.out = out;
+    do_xlate_actions(&ctx);
+}
+
 static int
 handle_packet_out(struct ofproto *p, struct ofconn *ofconn,
                   struct ofp_header *oh)
@@ -1196,6 +1287,7 @@ handle_packet_out(struct ofproto *p, struct ofconn *ofconn,
     struct odp_actions actions;
     int n_actions;
     uint16_t in_port;
+    flow_t flow;
     int error;
 
     error = check_ofp_packet_out(oh, &payload, &n_actions);
@@ -1215,10 +1307,10 @@ handle_packet_out(struct ofproto *p, struct ofconn *ofconn,
         buffer = NULL;
     }
 
-    in_port = ofp_port_to_odp_port(ntohs(opo->in_port));
-    ofp_actions_to_odp_actions(ntohs(opo->in_port), opo->actions,
-                               n_actions, &actions);
-    dpif_execute(&p->dpif, in_port, actions.actions, actions.n_actions,
+    flow_extract(&payload, ofp_port_to_odp_port(ntohs(opo->in_port)), &flow);
+    xlate_actions((const union ofp_action *) opo->actions, n_actions, &flow,
+                  p, &actions);
+    dpif_execute(&p->dpif, flow.in_port, actions.actions, actions.n_actions,
                  &payload);
     free_actions(&actions);
     ofpbuf_delete(buffer);
@@ -1670,33 +1762,31 @@ update_stats(struct rule *rule, const struct odp_flow_stats *stats)
 }
 
 static int
-send_buffered(struct ofproto *p, struct ofconn *ofconn,
-              struct ofp_flow_mod *ofm, size_t n_actions,
-              int *byte_count)
+send_buffered(struct ofproto *p, struct ofconn *ofconn, uint32_t buffer_id,
+              const struct rule *rule, int *byte_count)
 {
     struct odp_actions actions;
     struct ofpbuf *packet;
     uint16_t in_port;
+    flow_t flow;
     int error;
 
-    *byte_count = 0;
-    if (ofm->buffer_id == htonl(UINT32_MAX)) {
-        return 0;
-    } else if (!ofconn->pktbuf) {
+    if (!ofconn->pktbuf) {
         VLOG_WARN_RL(&rl, "attempt to send buffered packet via connection "
                      "without buffers");
         return ofp_mkerr(OFPET_BAD_REQUEST, OFPBRC_BAD_COOKIE);
     }
 
-    error = pktbuf_retrieve(ofconn->pktbuf, ntohl(ofm->buffer_id),
-                            &packet, &in_port);
+    error = pktbuf_retrieve(ofconn->pktbuf, buffer_id, &packet, &in_port);
     if (error) {
         return error;
     }
 
-    ofp_actions_to_odp_actions(in_port, ofm->actions, n_actions, &actions);
-    if (!dpif_execute(&p->dpif, ofp_port_to_odp_port(in_port),
-                      actions.actions, actions.n_actions, packet)) {
+    flow_extract(packet, in_port, &flow);
+    xlate_actions(rule->actions, rule->n_actions, &flow, p, &actions);
+    error = dpif_execute(&p->dpif, in_port,
+                         actions.actions, actions.n_actions, packet);
+    if (!error) {
         *byte_count = packet->size;
     }
     free_actions(&actions);
@@ -1710,18 +1800,15 @@ add_flow(struct ofproto *p, struct ofconn *ofconn,
          struct ofp_flow_mod *ofm, size_t n_actions)
 {
     struct rule *rule, *displaced_rule;
-    int byte_count;
     int buffer_error = 0;
 
-    buffer_error = send_buffered(p, ofconn, ofm, n_actions, &byte_count);
-
     rule = xmalloc(rule_size(n_actions));
     cls_rule_from_match(&rule->cr, &ofm->match, ntohs(ofm->priority));
     rule->idle_timeout = ntohs(ofm->idle_timeout);
     rule->hard_timeout = ntohs(ofm->hard_timeout);
     rule->used = rule->created = time_msec();
-    rule->packet_count = byte_count > 0;
-    rule->byte_count = byte_count;
+    rule->packet_count = 0;
+    rule->byte_count = 0;
     rule->tcp_flags = 0;
     rule->ip_tos = 0;
     rule->super = NULL;
@@ -1729,6 +1816,14 @@ add_flow(struct ofproto *p, struct ofconn *ofconn,
     rule->n_actions = n_actions;
     memcpy(rule->actions, ofm->actions, n_actions * sizeof *rule->actions);
 
+    if (ofm->buffer_id != htonl(UINT32_MAX)) {
+        int byte_count = 0;
+        buffer_error = send_buffered(p, ofconn, ntohl(ofm->buffer_id),
+                                     rule, &byte_count);
+        rule->byte_count += byte_count;
+        rule->packet_count += byte_count > 0;
+    }
+
     displaced_rule = rule_from_cls_rule(classifier_insert(&p->cls, &rule->cr));
     if (rule->cr.wc.wildcards) {
         if (displaced_rule) {
@@ -1747,10 +1842,8 @@ add_flow(struct ofproto *p, struct ofconn *ofconn,
                 struct odp_flow odp_flow;
 
                 subrule->super = rule;
-                ofp_actions_to_odp_actions(
-                    odp_port_to_ofp_port(subrule->cr.flow.in_port),
-                    (const struct ofp_action_header *) rule->actions,
-                    rule->n_actions, &actions);
+                xlate_actions(rule->actions, rule->n_actions,
+                              &subrule->cr.flow, p, &actions);
                 odp_flow.key = subrule->cr.flow;
                 odp_flow.actions = actions.actions;
                 odp_flow.n_actions = actions.n_actions;
@@ -1763,8 +1856,8 @@ add_flow(struct ofproto *p, struct ofconn *ofconn,
         struct odp_flow odp_flow;
         struct odp_actions actions;
 
-        ofp_actions_to_odp_actions(ntohs(ofm->match.in_port),
-                                   ofm->actions, n_actions, &actions);
+        xlate_actions((const union ofp_action *) ofm->actions, n_actions,
+                      &rule->cr.flow, p, &actions);
 
         odp_flow.key = rule->cr.flow;
         odp_flow.actions = actions.actions;
@@ -1804,8 +1897,8 @@ modify_flow(struct ofproto *p, const struct ofp_flow_mod *ofm,
             struct odp_flow odp_flow;
             struct odp_actions actions;
 
-            ofp_actions_to_odp_actions(rule->cr.flow.in_port,
-                                       ofm->actions, n_actions, &actions);
+            xlate_actions((const union ofp_action *) ofm->actions, n_actions,
+                          &rule->cr.flow, p, &actions);
             odp_flow.key = rule->cr.flow;
             odp_flow.actions = actions.actions;
             odp_flow.n_actions = actions.n_actions;
@@ -2105,7 +2198,7 @@ handle_odp_msg(struct ofproto *p, struct ofpbuf *packet)
                 free(subrule);
 
                 /* Execute old_sr on packet. */
-                rule_make_actions(old_sr, &actions);
+                rule_make_actions(p, old_sr, &actions);
                 dpif_execute(&p->dpif, msg->port,
                              actions.actions, actions.n_actions, &payload);
                 free_actions(&actions);
@@ -2121,7 +2214,7 @@ handle_odp_msg(struct ofproto *p, struct ofpbuf *packet)
         rule->used = time_msec();
 
         /* Install flow entry into datapath. */
-        rule_make_actions(subrule, &actions);
+        rule_make_actions(p, subrule, &actions);
         odp_flow.key = flow;
         odp_flow.actions = actions.actions;
         odp_flow.n_actions = actions.n_actions;
@@ -2131,7 +2224,7 @@ handle_odp_msg(struct ofproto *p, struct ofpbuf *packet)
          * collision?  Oh, it could also indicate that the packet was buffered
          * before we processed another packet from the same flow. */
         subrule = rule;
-        rule_make_actions(subrule, &actions);
+        rule_make_actions(p, subrule, &actions);
     }
 
     /* Execute subrule on packet. */
@@ -2142,24 +2235,30 @@ handle_odp_msg(struct ofproto *p, struct ofpbuf *packet)
 }
 \f
 static void
-revalidate_subrule(struct cls_rule *sub_, void *p_)
+revalidate_subrule_cb(struct cls_rule *sub_, void *p_)
 {
     struct rule *sub = rule_from_cls_rule(sub_);
     struct ofproto *p = p_;
-    struct rule *super;
 
-    if (!sub->super) {
-        /* Not a subrule. */
-        return;
+    if (sub->super) {
+        revalidate_subrule(p, sub);
     }
+}
+
+static bool
+revalidate_subrule(struct ofproto *p, struct rule *sub)
+{
+    const flow_t *flow = &sub->cr.flow;
+    struct rule *super;
 
-    super = rule_from_cls_rule(classifier_lookup_wild(&p->cls, &sub->cr.flow));
+    super = rule_from_cls_rule(classifier_lookup_wild(&p->cls, flow));
     if (super != sub->super) {
         if (!super) {
             struct odp_flow odp_flow;
             odp_flow.key = sub->cr.flow;
             dpif_flow_del(&p->dpif, &odp_flow);
             rule_destroy(sub);
+            return false;
         } else {
             struct odp_actions actions;
 
@@ -2169,12 +2268,13 @@ revalidate_subrule(struct cls_rule *sub_, void *p_)
             sub->created = super->created;
             sub->used = 0;
 
-            rule_make_actions(sub, &actions);
-            dpif_flow_set_actions(&p->dpif, &sub->cr.flow, actions.actions,
+            rule_make_actions(p, sub, &actions);
+            dpif_flow_set_actions(&p->dpif, flow, actions.actions,
                                   actions.n_actions);
             free_actions(&actions);
         }
     }
+    return true;
 }
 
 static struct ofpbuf *