- return 0;
-}
-
-static int do_put_flow(struct datapath *dp, struct odp_flow_put *uf,
- struct odp_flow_stats *stats)
-{
- struct tbl_node *flow_node;
- struct sw_flow *flow;
- struct tbl *table;
- int error;
-
- memset(uf->flow.key.reserved, 0, sizeof uf->flow.key.reserved);
-
- table = rcu_dereference(dp->table);
- flow_node = tbl_lookup(table, &uf->flow.key, flow_hash(&uf->flow.key), flow_cmp);
- if (!flow_node) {
- /* No such flow. */
- struct sw_flow_actions *acts;
-
- error = -ENOENT;
- if (!(uf->flags & ODPPF_CREATE))
- goto error;
-
- /* Expand table, if necessary, to make room. */
- if (tbl_count(table) >= tbl_n_buckets(table)) {
- error = expand_table(dp);
- if (error)
- goto error;
- table = rcu_dereference(dp->table);
- }
-
- /* Allocate flow. */
- error = -ENOMEM;
- flow = kmem_cache_alloc(flow_cache, GFP_KERNEL);
- if (flow == NULL)
- goto error;
- flow->key = uf->flow.key;
- spin_lock_init(&flow->lock);
- clear_stats(flow);
-
- /* Obtain actions. */
- acts = get_actions(&uf->flow);
- error = PTR_ERR(acts);
- if (IS_ERR(acts))
- goto error_free_flow;
- rcu_assign_pointer(flow->sf_acts, acts);
-
- /* Put flow in bucket. */
- error = tbl_insert(table, &flow->tbl_node, flow_hash(&flow->key));
- if (error)
- goto error_free_flow_acts;
-
- memset(stats, 0, sizeof(struct odp_flow_stats));
- } else {
- /* We found a matching flow. */
- struct sw_flow_actions *old_acts, *new_acts;
-
- flow = flow_cast(flow_node);
-
- /* Bail out if we're not allowed to modify an existing flow. */
- error = -EEXIST;
- if (!(uf->flags & ODPPF_MODIFY))
- goto error;
-
- /* Swap actions. */
- new_acts = get_actions(&uf->flow);
- error = PTR_ERR(new_acts);
- if (IS_ERR(new_acts))
- goto error;
- old_acts = rcu_dereference(flow->sf_acts);
- if (old_acts->n_actions != new_acts->n_actions ||
- memcmp(old_acts->actions, new_acts->actions,
- sizeof(union odp_action) * old_acts->n_actions)) {
- rcu_assign_pointer(flow->sf_acts, new_acts);
- flow_deferred_free_acts(old_acts);
- } else {
- kfree(new_acts);
- }
-
- /* Fetch stats, then clear them if necessary. */
- spin_lock_bh(&flow->lock);
- get_stats(flow, stats, get_time_offset());
- if (uf->flags & ODPPF_ZERO_STATS)
- clear_stats(flow);
- spin_unlock_bh(&flow->lock);
- }
-
- return 0;
-
-error_free_flow_acts:
- kfree(flow->sf_acts);
-error_free_flow:
- kmem_cache_free(flow_cache, flow);
-error:
- return error;
-}
-
-static int put_flow(struct datapath *dp, struct odp_flow_put __user *ufp)
-{
- struct odp_flow_stats stats;
- struct odp_flow_put uf;
- int error;
-
- if (copy_from_user(&uf, ufp, sizeof(struct odp_flow_put)))
- return -EFAULT;
-
- error = do_put_flow(dp, &uf, &stats);
- if (error)
- return error;
-
- if (copy_to_user(&ufp->flow.stats, &stats,
- sizeof(struct odp_flow_stats)))
- return -EFAULT;
-
- return 0;
-}
-
-static int do_answer_query(struct sw_flow *flow, u32 query_flags,
- struct timespec time_offset,
- struct odp_flow_stats __user *ustats,
- union odp_action __user *actions,
- u32 __user *n_actionsp)
-{
- struct sw_flow_actions *sf_acts;
- struct odp_flow_stats stats;
- u32 n_actions;
-
- spin_lock_bh(&flow->lock);
- get_stats(flow, &stats, time_offset);
- if (query_flags & ODPFF_ZERO_TCP_FLAGS)
- flow->tcp_flags = 0;
-
- spin_unlock_bh(&flow->lock);
-
- if (copy_to_user(ustats, &stats, sizeof(struct odp_flow_stats)) ||
- get_user(n_actions, n_actionsp))
- return -EFAULT;
-
- if (!n_actions)
- return 0;
-
- sf_acts = rcu_dereference(flow->sf_acts);
- if (put_user(sf_acts->n_actions, n_actionsp) ||
- (actions && copy_to_user(actions, sf_acts->actions,
- sizeof(union odp_action) *
- min(sf_acts->n_actions, n_actions))))
- return -EFAULT;
-
- return 0;
-}
-
-static int answer_query(struct sw_flow *flow, u32 query_flags,
- struct timespec time_offset,
- struct odp_flow __user *ufp)
-{
- union odp_action *actions;
-
- if (get_user(actions, &ufp->actions))
- return -EFAULT;
-
- return do_answer_query(flow, query_flags, time_offset,
- &ufp->stats, actions, &ufp->n_actions);
-}
-
-static struct sw_flow *do_del_flow(struct datapath *dp, struct odp_flow_key *key)
-{
- struct tbl *table = rcu_dereference(dp->table);
- struct tbl_node *flow_node;
- int error;
-
- memset(key->reserved, 0, sizeof key->reserved);
- flow_node = tbl_lookup(table, key, flow_hash(key), flow_cmp);
- if (!flow_node)
- return ERR_PTR(-ENOENT);
-
- error = tbl_remove(table, flow_node);
- if (error)
- return ERR_PTR(error);
-
- /* XXX Returned flow_node's statistics might lose a few packets, since
- * other CPUs can be using this flow. We used to synchronize_rcu() to
- * make sure that we get completely accurate stats, but that blows our
- * performance, badly. */
- return flow_cast(flow_node);
-}
-
-static int del_flow(struct datapath *dp, struct odp_flow __user *ufp)
-{
- struct sw_flow *flow;
- struct odp_flow uf;
- int error;
-
- if (copy_from_user(&uf, ufp, sizeof uf))
- return -EFAULT;
-
- flow = do_del_flow(dp, &uf.key);
- if (IS_ERR(flow))
- return PTR_ERR(flow);
-
- error = answer_query(flow, 0, get_time_offset(), ufp);
- flow_deferred_free(flow);
- return error;
-}
-
-static int do_query_flows(struct datapath *dp, const struct odp_flowvec *flowvec)
-{
- struct tbl *table = rcu_dereference(dp->table);
- struct timespec time_offset;
- u32 i;
-
- time_offset = get_time_offset();
-
- for (i = 0; i < flowvec->n_flows; i++) {
- struct odp_flow __user *ufp = &flowvec->flows[i];
- struct odp_flow uf;
- struct tbl_node *flow_node;
- int error;
-
- if (copy_from_user(&uf, ufp, sizeof uf))
- return -EFAULT;
- memset(uf.key.reserved, 0, sizeof uf.key.reserved);
-
- flow_node = tbl_lookup(table, &uf.key, flow_hash(&uf.key), flow_cmp);
- if (!flow_node)
- error = put_user(ENOENT, &ufp->stats.error);
- else
- error = answer_query(flow_cast(flow_node), uf.flags, time_offset, ufp);
- if (error)
- return -EFAULT;
- }
- return flowvec->n_flows;
-}
-
-struct list_flows_cbdata {
- struct odp_flow __user *uflows;
- u32 n_flows;
- u32 listed_flows;
- struct timespec time_offset;
-};
-
-static int list_flow(struct tbl_node *node, void *cbdata_)
-{
- struct sw_flow *flow = flow_cast(node);
- struct list_flows_cbdata *cbdata = cbdata_;
- struct odp_flow __user *ufp = &cbdata->uflows[cbdata->listed_flows++];
- int error;
-
- if (copy_to_user(&ufp->key, &flow->key, sizeof flow->key))
- return -EFAULT;
- error = answer_query(flow, 0, cbdata->time_offset, ufp);
- if (error)
- return error;
-
- if (cbdata->listed_flows >= cbdata->n_flows)
- return cbdata->listed_flows;
- return 0;
-}
-
-static int do_list_flows(struct datapath *dp, const struct odp_flowvec *flowvec)
-{
- struct list_flows_cbdata cbdata;
- int error;
-
- if (!flowvec->n_flows)
- return 0;
-
- cbdata.uflows = flowvec->flows;
- cbdata.n_flows = flowvec->n_flows;
- cbdata.listed_flows = 0;
- cbdata.time_offset = get_time_offset();
-
- error = tbl_foreach(rcu_dereference(dp->table), list_flow, &cbdata);
- return error ? error : cbdata.listed_flows;
-}
-
-static int do_flowvec_ioctl(struct datapath *dp, unsigned long argp,
- int (*function)(struct datapath *,
- const struct odp_flowvec *))
-{
- struct odp_flowvec __user *uflowvec;
- struct odp_flowvec flowvec;
- int retval;
-
- uflowvec = (struct odp_flowvec __user *)argp;
- if (copy_from_user(&flowvec, uflowvec, sizeof flowvec))
- return -EFAULT;
-
- if (flowvec.n_flows > INT_MAX / sizeof(struct odp_flow))
- return -EINVAL;
-
- retval = function(dp, &flowvec);
- return (retval < 0 ? retval
- : retval == flowvec.n_flows ? 0
- : put_user(retval, &uflowvec->n_flows));