datapath: Eliminate synchronize_rcu() in table swap.

We found out some time ago that synchronize_rcu() can block for multiple
seconds in some cases, so it's a good idea to eliminate as many of them
as we can.

This commit eliminates a call to synchronize_rcu() from functions that
expand or flush flow tables.  To avoid adding a member to dp_table that
specifies the "free_flows" argument to dp_table_destroy(), the commit
uses two different callback functions and manually inlines dp_table_swap()
into its callers.

Bug #1233.

diff --git a/datapath/datapath.h b/datapath/datapath.h
index 43914263fdf8b5313d5a368c05e3f7027ad773ee..bb9ce0a331b9ba43e33276de4c5a81649456d573 100644 (file)
--- a/datapath/datapath.h
+++ b/datapath/datapath.h
@@ -34,6 +34,7 @@ struct sk_buff;
 struct dp_table {
        unsigned int n_buckets;
        struct sw_flow ***flows[2];
+       struct rcu_head rcu;
 };
 
 #define DP_N_QUEUES 2

diff --git a/datapath/table.c b/datapath/table.c
index b78f3b2ffe1741b219ea033c06d084a524c8d627..7890442d5840eed0a81a3fc66b67e05ec37d8c29 100644 (file)
--- a/datapath/table.c
+++ b/datapath/table.c
@@ -127,15 +127,6 @@ struct sw_flow *dp_table_lookup(struct dp_table *table,
        return flow;
 }
 
-static void dp_table_swap(struct datapath *dp, struct dp_table *new_table,
-                         int free_flows)
-{
-       struct dp_table *old_table = rcu_dereference(dp->table);
-       rcu_assign_pointer(dp->table, new_table);
-       synchronize_rcu();
-       dp_table_destroy(old_table, free_flows);
-}
-
 int dp_table_foreach(struct dp_table *table,
                     int (*callback)(struct sw_flow *flow, void *aux),
                     void *aux)
@@ -174,6 +165,12 @@ static int insert_flow(struct sw_flow *flow, void *new_table_)
        return 0;
 }
 
+static void dp_free_table_rcu(struct rcu_head *rcu)
+{
+       struct dp_table *table = container_of(rcu, struct dp_table, rcu);
+       dp_table_destroy(table, 0);
+}
+
 int dp_table_expand(struct datapath *dp)
 {
        struct dp_table *old_table = rcu_dereference(dp->table);
@@ -181,16 +178,25 @@ int dp_table_expand(struct datapath *dp)
        if (!new_table)
                return -ENOMEM;
        dp_table_foreach(old_table, insert_flow, new_table);
-       dp_table_swap(dp, new_table, 0);
+       rcu_assign_pointer(dp->table, new_table);
+       call_rcu(&old_table->rcu, dp_free_table_rcu);
        return 0;
 }
 
+static void dp_free_table_and_flows_rcu(struct rcu_head *rcu)
+{
+       struct dp_table *table = container_of(rcu, struct dp_table, rcu);
+       dp_table_destroy(table, 1);
+}
+
 int dp_table_flush(struct datapath *dp)
 {
+       struct dp_table *old_table = rcu_dereference(dp->table);
        struct dp_table *new_table = dp_table_create(DP_L1_SIZE);
        if (!new_table)
                return -ENOMEM;
-       dp_table_swap(dp, new_table, 1);
+       rcu_assign_pointer(dp->table, new_table);
+       call_rcu(&old_table->rcu, dp_free_table_and_flows_rcu);
        return 0;
 }