+static const struct nla_policy flow_policy[ODP_FLOW_ATTR_MAX + 1] = {
+ [ODP_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
+ [ODP_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
+ [ODP_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
+};
+
+static struct genl_family dp_flow_genl_family = {
+ .id = GENL_ID_GENERATE,
+ .hdrsize = sizeof(struct odp_header),
+ .name = ODP_FLOW_FAMILY,
+ .version = 1,
+ .maxattr = ODP_FLOW_ATTR_MAX
+};
+
+static struct genl_multicast_group dp_flow_multicast_group = {
+ .name = ODP_FLOW_MCGROUP
+};
+
+/* Called with genl_lock. */
+static int odp_flow_cmd_fill_info(struct sw_flow *flow, struct datapath *dp,
+ struct sk_buff *skb, u32 pid, u32 seq, u32 flags, u8 cmd)
+{
+ const int skb_orig_len = skb->len;
+ const struct sw_flow_actions *sf_acts;
+ struct odp_flow_stats stats;
+ struct odp_header *odp_header;
+ struct nlattr *nla;
+ unsigned long used;
+ u8 tcp_flags;
+ int err;
+
+ sf_acts = rcu_dereference_protected(flow->sf_acts,
+ lockdep_genl_is_held());
+
+ odp_header = genlmsg_put(skb, pid, seq, &dp_flow_genl_family, flags, cmd);
+ if (!odp_header)
+ return -EMSGSIZE;
+
+ odp_header->dp_ifindex = dp->dp_ifindex;
+
+ nla = nla_nest_start(skb, ODP_FLOW_ATTR_KEY);
+ if (!nla)
+ goto nla_put_failure;
+ err = flow_to_nlattrs(&flow->key, skb);
+ if (err)
+ goto error;
+ nla_nest_end(skb, nla);
+
+ spin_lock_bh(&flow->lock);
+ used = flow->used;
+ stats.n_packets = flow->packet_count;
+ stats.n_bytes = flow->byte_count;
+ tcp_flags = flow->tcp_flags;
+ spin_unlock_bh(&flow->lock);
+
+ if (used)
+ NLA_PUT_U64(skb, ODP_FLOW_ATTR_USED, flow_used_time(used));
+
+ if (stats.n_packets)
+ NLA_PUT(skb, ODP_FLOW_ATTR_STATS, sizeof(struct odp_flow_stats), &stats);
+
+ if (tcp_flags)
+ NLA_PUT_U8(skb, ODP_FLOW_ATTR_TCP_FLAGS, tcp_flags);
+
+ /* If ODP_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
+ * this is the first flow to be dumped into 'skb'. This is unusual for
+ * Netlink but individual action lists can be longer than
+ * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
+ * The userspace caller can always fetch the actions separately if it
+ * really wants them. (Most userspace callers in fact don't care.)
+ *
+ * This can only fail for dump operations because the skb is always
+ * properly sized for single flows.
+ */
+ err = nla_put(skb, ODP_FLOW_ATTR_ACTIONS, sf_acts->actions_len,
+ sf_acts->actions);
+ if (err < 0 && skb_orig_len)
+ goto error;
+
+ return genlmsg_end(skb, odp_header);
+
+nla_put_failure:
+ err = -EMSGSIZE;
+error:
+ genlmsg_cancel(skb, odp_header);
+ return err;
+}
+
+static struct sk_buff *odp_flow_cmd_alloc_info(struct sw_flow *flow)
+{
+ const struct sw_flow_actions *sf_acts;
+ int len;
+
+ sf_acts = rcu_dereference_protected(flow->sf_acts,
+ lockdep_genl_is_held());
+
+ len = nla_total_size(FLOW_BUFSIZE); /* ODP_FLOW_ATTR_KEY */
+ len += nla_total_size(sf_acts->actions_len); /* ODP_FLOW_ATTR_ACTIONS */
+ len += nla_total_size(sizeof(struct odp_flow_stats)); /* ODP_FLOW_ATTR_STATS */
+ len += nla_total_size(1); /* ODP_FLOW_ATTR_TCP_FLAGS */
+ len += nla_total_size(8); /* ODP_FLOW_ATTR_USED */
+ return genlmsg_new(NLMSG_ALIGN(sizeof(struct odp_header)) + len, GFP_KERNEL);
+}
+
+static struct sk_buff *odp_flow_cmd_build_info(struct sw_flow *flow, struct datapath *dp,
+ u32 pid, u32 seq, u8 cmd)
+{
+ struct sk_buff *skb;
+ int retval;
+
+ skb = odp_flow_cmd_alloc_info(flow);
+ if (!skb)
+ return ERR_PTR(-ENOMEM);
+
+ retval = odp_flow_cmd_fill_info(flow, dp, skb, pid, seq, 0, cmd);
+ BUG_ON(retval < 0);
+ return skb;
+}
+
+static int odp_flow_cmd_new_or_set(struct sk_buff *skb, struct genl_info *info)
+{
+ struct nlattr **a = info->attrs;
+ struct odp_header *odp_header = info->userhdr;
+ struct tbl_node *flow_node;
+ struct sw_flow_key key;
+ struct sw_flow *flow;
+ struct sk_buff *reply;
+ struct datapath *dp;
+ struct tbl *table;
+ u32 hash;
+ int error;
+ int key_len;
+
+ /* Extract key. */
+ error = -EINVAL;
+ if (!a[ODP_FLOW_ATTR_KEY])
+ goto error;
+ error = flow_from_nlattrs(&key, &key_len, a[ODP_FLOW_ATTR_KEY]);
+ if (error)
+ goto error;
+
+ /* Validate actions. */
+ if (a[ODP_FLOW_ATTR_ACTIONS]) {
+ error = validate_actions(a[ODP_FLOW_ATTR_ACTIONS]);
+ if (error)
+ goto error;
+ } else if (info->genlhdr->cmd == ODP_FLOW_CMD_NEW) {
+ error = -EINVAL;
+ goto error;
+ }
+
+ dp = get_dp(odp_header->dp_ifindex);
+ error = -ENODEV;
+ if (!dp)
+ goto error;
+
+ hash = flow_hash(&key, key_len);
+ table = get_table_protected(dp);
+ flow_node = tbl_lookup(table, &key, key_len, hash, flow_cmp);
+ if (!flow_node) {
+ struct sw_flow_actions *acts;
+
+ /* Bail out if we're not allowed to create a new flow. */
+ error = -ENOENT;
+ if (info->genlhdr->cmd == ODP_FLOW_CMD_SET)
+ goto error;
+
+ /* Expand table, if necessary, to make room. */
+ if (tbl_count(table) >= tbl_n_buckets(table)) {
+ error = expand_table(dp);
+ if (error)
+ goto error;
+ table = get_table_protected(dp);
+ }
+
+ /* Allocate flow. */
+ flow = flow_alloc();
+ if (IS_ERR(flow)) {
+ error = PTR_ERR(flow);
+ goto error;
+ }
+ flow->key = key;
+ clear_stats(flow);
+
+ /* Obtain actions. */
+ acts = flow_actions_alloc(a[ODP_FLOW_ATTR_ACTIONS]);
+ error = PTR_ERR(acts);
+ if (IS_ERR(acts))
+ goto error_free_flow;
+ rcu_assign_pointer(flow->sf_acts, acts);
+
+ /* Put flow in bucket. */
+ error = tbl_insert(table, &flow->tbl_node, hash);
+ if (error)
+ goto error_free_flow;
+
+ reply = odp_flow_cmd_build_info(flow, dp, info->snd_pid,
+ info->snd_seq, ODP_FLOW_CMD_NEW);
+ } else {
+ /* We found a matching flow. */
+ struct sw_flow_actions *old_acts;
+
+ /* Bail out if we're not allowed to modify an existing flow.
+ * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
+ * because Generic Netlink treats the latter as a dump
+ * request. We also accept NLM_F_EXCL in case that bug ever
+ * gets fixed.
+ */
+ error = -EEXIST;
+ if (info->genlhdr->cmd == ODP_FLOW_CMD_NEW &&
+ info->nlhdr->nlmsg_flags & (NLM_F_CREATE | NLM_F_EXCL))
+ goto error;
+
+ /* Update actions. */
+ flow = flow_cast(flow_node);
+ old_acts = rcu_dereference_protected(flow->sf_acts,
+ lockdep_genl_is_held());
+ if (a[ODP_FLOW_ATTR_ACTIONS] &&
+ (old_acts->actions_len != nla_len(a[ODP_FLOW_ATTR_ACTIONS]) ||
+ memcmp(old_acts->actions, nla_data(a[ODP_FLOW_ATTR_ACTIONS]),
+ old_acts->actions_len))) {
+ struct sw_flow_actions *new_acts;
+
+ new_acts = flow_actions_alloc(a[ODP_FLOW_ATTR_ACTIONS]);
+ error = PTR_ERR(new_acts);
+ if (IS_ERR(new_acts))
+ goto error;
+
+ rcu_assign_pointer(flow->sf_acts, new_acts);
+ flow_deferred_free_acts(old_acts);
+ }
+
+ reply = odp_flow_cmd_build_info(flow, dp, info->snd_pid,
+ info->snd_seq, ODP_FLOW_CMD_NEW);
+
+ /* Clear stats. */
+ if (a[ODP_FLOW_ATTR_CLEAR]) {
+ spin_lock_bh(&flow->lock);
+ clear_stats(flow);
+ spin_unlock_bh(&flow->lock);
+ }
+ }
+
+ if (!IS_ERR(reply))
+ genl_notify(reply, genl_info_net(info), info->snd_pid,
+ dp_flow_multicast_group.id, info->nlhdr, GFP_KERNEL);
+ else
+ netlink_set_err(INIT_NET_GENL_SOCK, 0,
+ dp_flow_multicast_group.id, PTR_ERR(reply));
+ return 0;
+
+error_free_flow:
+ flow_put(flow);
+error:
+ return error;
+}
+
+static int odp_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
+{
+ struct nlattr **a = info->attrs;
+ struct odp_header *odp_header = info->userhdr;
+ struct sw_flow_key key;
+ struct tbl_node *flow_node;
+ struct sk_buff *reply;
+ struct sw_flow *flow;
+ struct datapath *dp;
+ struct tbl *table;
+ int err;
+ int key_len;
+
+ if (!a[ODP_FLOW_ATTR_KEY])
+ return -EINVAL;
+ err = flow_from_nlattrs(&key, &key_len, a[ODP_FLOW_ATTR_KEY]);
+ if (err)
+ return err;
+
+ dp = get_dp(odp_header->dp_ifindex);
+ if (!dp)
+ return -ENODEV;
+
+ table = get_table_protected(dp);
+ flow_node = tbl_lookup(table, &key, key_len, flow_hash(&key, key_len),
+ flow_cmp);
+ if (!flow_node)
+ return -ENOENT;
+
+ flow = flow_cast(flow_node);
+ reply = odp_flow_cmd_build_info(flow, dp, info->snd_pid, info->snd_seq, ODP_FLOW_CMD_NEW);
+ if (IS_ERR(reply))
+ return PTR_ERR(reply);
+
+ return genlmsg_reply(reply, info);
+}
+
+static int odp_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
+{
+ struct nlattr **a = info->attrs;
+ struct odp_header *odp_header = info->userhdr;
+ struct sw_flow_key key;
+ struct tbl_node *flow_node;
+ struct sk_buff *reply;
+ struct sw_flow *flow;
+ struct datapath *dp;
+ struct tbl *table;
+ int err;
+ int key_len;
+
+ if (!a[ODP_FLOW_ATTR_KEY])
+ return flush_flows(odp_header->dp_ifindex);
+ err = flow_from_nlattrs(&key, &key_len, a[ODP_FLOW_ATTR_KEY]);
+ if (err)
+ return err;
+
+ dp = get_dp(odp_header->dp_ifindex);
+ if (!dp)
+ return -ENODEV;
+
+ table = get_table_protected(dp);
+ flow_node = tbl_lookup(table, &key, key_len, flow_hash(&key, key_len),
+ flow_cmp);
+ if (!flow_node)
+ return -ENOENT;
+ flow = flow_cast(flow_node);
+
+ reply = odp_flow_cmd_alloc_info(flow);
+ if (!reply)
+ return -ENOMEM;
+
+ err = tbl_remove(table, flow_node);
+ if (err) {
+ kfree_skb(reply);
+ return err;
+ }
+
+ err = odp_flow_cmd_fill_info(flow, dp, reply, info->snd_pid,
+ info->snd_seq, 0, ODP_FLOW_CMD_DEL);
+ BUG_ON(err < 0);
+
+ flow_deferred_free(flow);
+
+ genl_notify(reply, genl_info_net(info), info->snd_pid,
+ dp_flow_multicast_group.id, info->nlhdr, GFP_KERNEL);
+ return 0;
+}
+
+static int odp_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
+{
+ struct odp_header *odp_header = genlmsg_data(nlmsg_data(cb->nlh));
+ struct datapath *dp;
+
+ dp = get_dp(odp_header->dp_ifindex);
+ if (!dp)
+ return -ENODEV;
+
+ for (;;) {
+ struct tbl_node *flow_node;
+ struct sw_flow *flow;
+ u32 bucket, obj;
+
+ bucket = cb->args[0];
+ obj = cb->args[1];
+ flow_node = tbl_next(get_table_protected(dp), &bucket, &obj);
+ if (!flow_node)
+ break;
+
+ flow = flow_cast(flow_node);
+ if (odp_flow_cmd_fill_info(flow, dp, skb, NETLINK_CB(cb->skb).pid,
+ cb->nlh->nlmsg_seq, NLM_F_MULTI,
+ ODP_FLOW_CMD_NEW) < 0)
+ break;
+
+ cb->args[0] = bucket;
+ cb->args[1] = obj;
+ }
+ return skb->len;
+}
+
+static struct genl_ops dp_flow_genl_ops[] = {
+ { .cmd = ODP_FLOW_CMD_NEW,
+ .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
+ .policy = flow_policy,
+ .doit = odp_flow_cmd_new_or_set
+ },
+ { .cmd = ODP_FLOW_CMD_DEL,
+ .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
+ .policy = flow_policy,
+ .doit = odp_flow_cmd_del
+ },
+ { .cmd = ODP_FLOW_CMD_GET,
+ .flags = 0, /* OK for unprivileged users. */
+ .policy = flow_policy,
+ .doit = odp_flow_cmd_get,
+ .dumpit = odp_flow_cmd_dump
+ },
+ { .cmd = ODP_FLOW_CMD_SET,
+ .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
+ .policy = flow_policy,
+ .doit = odp_flow_cmd_new_or_set,
+ },
+};
+
+static const struct nla_policy datapath_policy[ODP_DP_ATTR_MAX + 1] = {
+#ifdef HAVE_NLA_NUL_STRING
+ [ODP_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
+#endif
+ [ODP_DP_ATTR_IPV4_FRAGS] = { .type = NLA_U32 },
+ [ODP_DP_ATTR_SAMPLING] = { .type = NLA_U32 },
+};
+
+static struct genl_family dp_datapath_genl_family = {
+ .id = GENL_ID_GENERATE,
+ .hdrsize = sizeof(struct odp_header),
+ .name = ODP_DATAPATH_FAMILY,
+ .version = 1,
+ .maxattr = ODP_DP_ATTR_MAX
+};
+
+static struct genl_multicast_group dp_datapath_multicast_group = {
+ .name = ODP_DATAPATH_MCGROUP
+};
+
+static int odp_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
+ u32 pid, u32 seq, u32 flags, u8 cmd)
+{
+ struct odp_header *odp_header;
+ struct nlattr *nla;
+ int err;
+
+ odp_header = genlmsg_put(skb, pid, seq, &dp_datapath_genl_family,
+ flags, cmd);
+ if (!odp_header)
+ goto error;
+
+ odp_header->dp_ifindex = dp->dp_ifindex;
+