Commit 80cd22c3 authored by Paul Blakey's avatar Paul Blakey Committed by Jakub Kicinski

net/sched: cls_api: Support hardware miss to tc action

For drivers to support partial offload of a filter's action list,
add support for action miss to specify an action instance to
continue from in sw.

CT action in particular can't be fully offloaded, as new connections
need to be handled in software. This imposes other limitations on
the actions that can be offloaded together with the CT action, such
as packet modifications.

Assign each action on a filter's action list a unique miss_cookie
which drivers can then use to fill action_miss part of the tc skb
extension. On getting back this miss_cookie, find the action
instance with relevant cookie and continue classifying from there.
Signed-off-by: default avatarPaul Blakey <paulb@nvidia.com>
Reviewed-by: default avatarJiri Pirko <jiri@nvidia.com>
Reviewed-by: default avatarSimon Horman <simon.horman@corigine.com>
Reviewed-by: default avatarMarcelo Ricardo Leitner <marcelo.leitner@gmail.com>
Acked-by: default avatarJamal Hadi Salim <jhs@mojatatu.com>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent db4b4902
...@@ -319,12 +319,16 @@ struct nf_bridge_info { ...@@ -319,12 +319,16 @@ struct nf_bridge_info {
* and read by ovs to recirc_id. * and read by ovs to recirc_id.
*/ */
struct tc_skb_ext { struct tc_skb_ext {
union {
u64 act_miss_cookie;
__u32 chain; __u32 chain;
};
__u16 mru; __u16 mru;
__u16 zone; __u16 zone;
u8 post_ct:1; u8 post_ct:1;
u8 post_ct_snat:1; u8 post_ct_snat:1;
u8 post_ct_dnat:1; u8 post_ct_dnat:1;
u8 act_miss:1; /* Set if act_miss_cookie is used */
}; };
#endif #endif
......
...@@ -229,6 +229,7 @@ struct flow_action_entry { ...@@ -229,6 +229,7 @@ struct flow_action_entry {
enum flow_action_id id; enum flow_action_id id;
u32 hw_index; u32 hw_index;
unsigned long cookie; unsigned long cookie;
u64 miss_cookie;
enum flow_action_hw_stats hw_stats; enum flow_action_hw_stats hw_stats;
action_destr destructor; action_destr destructor;
void *destructor_priv; void *destructor_priv;
......
...@@ -59,6 +59,8 @@ int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q, ...@@ -59,6 +59,8 @@ int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
void tcf_block_put(struct tcf_block *block); void tcf_block_put(struct tcf_block *block);
void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q, void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
struct tcf_block_ext_info *ei); struct tcf_block_ext_info *ei);
int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action,
int police, struct tcf_proto *tp, u32 handle, bool used_action_miss);
static inline bool tcf_block_shared(struct tcf_block *block) static inline bool tcf_block_shared(struct tcf_block *block)
{ {
...@@ -229,6 +231,7 @@ struct tcf_exts { ...@@ -229,6 +231,7 @@ struct tcf_exts {
struct tc_action **actions; struct tc_action **actions;
struct net *net; struct net *net;
netns_tracker ns_tracker; netns_tracker ns_tracker;
struct tcf_exts_miss_cookie_node *miss_cookie_node;
#endif #endif
/* Map to export classifier specific extension TLV types to the /* Map to export classifier specific extension TLV types to the
* generic extensions API. Unsupported extensions must be set to 0. * generic extensions API. Unsupported extensions must be set to 0.
...@@ -240,21 +243,11 @@ struct tcf_exts { ...@@ -240,21 +243,11 @@ struct tcf_exts {
static inline int tcf_exts_init(struct tcf_exts *exts, struct net *net, static inline int tcf_exts_init(struct tcf_exts *exts, struct net *net,
int action, int police) int action, int police)
{ {
#ifdef CONFIG_NET_CLS_ACT #ifdef CONFIG_NET_CLS
exts->type = 0; return tcf_exts_init_ex(exts, net, action, police, NULL, 0, false);
exts->nr_actions = 0; #else
/* Note: we do not own yet a reference on net. return -EOPNOTSUPP;
* This reference might be taken later from tcf_exts_get_net().
*/
exts->net = net;
exts->actions = kcalloc(TCA_ACT_MAX_PRIO, sizeof(struct tc_action *),
GFP_KERNEL);
if (!exts->actions)
return -ENOMEM;
#endif #endif
exts->action = action;
exts->police = police;
return 0;
} }
/* Return false if the netns is being destroyed in cleanup_net(). Callers /* Return false if the netns is being destroyed in cleanup_net(). Callers
...@@ -360,6 +353,18 @@ tcf_exts_exec(struct sk_buff *skb, struct tcf_exts *exts, ...@@ -360,6 +353,18 @@ tcf_exts_exec(struct sk_buff *skb, struct tcf_exts *exts,
return TC_ACT_OK; return TC_ACT_OK;
} }
static inline int
tcf_exts_exec_ex(struct sk_buff *skb, struct tcf_exts *exts, int act_index,
struct tcf_result *res)
{
#ifdef CONFIG_NET_CLS_ACT
return tcf_action_exec(skb, exts->actions + act_index,
exts->nr_actions - act_index, res);
#else
return TC_ACT_OK;
#endif
}
int tcf_exts_validate(struct net *net, struct tcf_proto *tp, int tcf_exts_validate(struct net *net, struct tcf_proto *tp,
struct nlattr **tb, struct nlattr *rate_tlv, struct nlattr **tb, struct nlattr *rate_tlv,
struct tcf_exts *exts, u32 flags, struct tcf_exts *exts, u32 flags,
...@@ -584,6 +589,7 @@ int tc_setup_offload_action(struct flow_action *flow_action, ...@@ -584,6 +589,7 @@ int tc_setup_offload_action(struct flow_action *flow_action,
void tc_cleanup_offload_action(struct flow_action *flow_action); void tc_cleanup_offload_action(struct flow_action *flow_action);
int tc_setup_action(struct flow_action *flow_action, int tc_setup_action(struct flow_action *flow_action,
struct tc_action *actions[], struct tc_action *actions[],
u32 miss_cookie_base,
struct netlink_ext_ack *extack); struct netlink_ext_ack *extack);
int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type, int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
......
...@@ -369,6 +369,8 @@ struct tcf_proto_ops { ...@@ -369,6 +369,8 @@ struct tcf_proto_ops {
struct nlattr **tca, struct nlattr **tca,
struct netlink_ext_ack *extack); struct netlink_ext_ack *extack);
void (*tmplt_destroy)(void *tmplt_priv); void (*tmplt_destroy)(void *tmplt_priv);
struct tcf_exts * (*get_exts)(const struct tcf_proto *tp,
u32 handle);
/* rtnetlink specific */ /* rtnetlink specific */
int (*dump)(struct net*, struct tcf_proto*, void *, int (*dump)(struct net*, struct tcf_proto*, void *,
......
...@@ -1041,7 +1041,8 @@ int ovs_flow_key_extract(const struct ip_tunnel_info *tun_info, ...@@ -1041,7 +1041,8 @@ int ovs_flow_key_extract(const struct ip_tunnel_info *tun_info,
#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
if (tc_skb_ext_tc_enabled()) { if (tc_skb_ext_tc_enabled()) {
tc_ext = skb_ext_find(skb, TC_SKB_EXT); tc_ext = skb_ext_find(skb, TC_SKB_EXT);
key->recirc_id = tc_ext ? tc_ext->chain : 0; key->recirc_id = tc_ext && !tc_ext->act_miss ?
tc_ext->chain : 0;
OVS_CB(skb)->mru = tc_ext ? tc_ext->mru : 0; OVS_CB(skb)->mru = tc_ext ? tc_ext->mru : 0;
post_ct = tc_ext ? tc_ext->post_ct : false; post_ct = tc_ext ? tc_ext->post_ct : false;
post_ct_snat = post_ct ? tc_ext->post_ct_snat : false; post_ct_snat = post_ct ? tc_ext->post_ct_snat : false;
......
...@@ -268,7 +268,7 @@ static int tcf_action_offload_add_ex(struct tc_action *action, ...@@ -268,7 +268,7 @@ static int tcf_action_offload_add_ex(struct tc_action *action,
if (err) if (err)
goto fl_err; goto fl_err;
err = tc_setup_action(&fl_action->action, actions, extack); err = tc_setup_action(&fl_action->action, actions, 0, extack);
if (err) { if (err) {
NL_SET_ERR_MSG_MOD(extack, NL_SET_ERR_MSG_MOD(extack,
"Failed to setup tc actions for offload"); "Failed to setup tc actions for offload");
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#include <linux/idr.h> #include <linux/idr.h>
#include <linux/jhash.h> #include <linux/jhash.h>
#include <linux/rculist.h> #include <linux/rculist.h>
#include <linux/rhashtable.h>
#include <net/net_namespace.h> #include <net/net_namespace.h>
#include <net/sock.h> #include <net/sock.h>
#include <net/netlink.h> #include <net/netlink.h>
...@@ -50,6 +51,109 @@ static LIST_HEAD(tcf_proto_base); ...@@ -50,6 +51,109 @@ static LIST_HEAD(tcf_proto_base);
/* Protects list of registered TC modules. It is pure SMP lock. */ /* Protects list of registered TC modules. It is pure SMP lock. */
static DEFINE_RWLOCK(cls_mod_lock); static DEFINE_RWLOCK(cls_mod_lock);
static struct xarray tcf_exts_miss_cookies_xa;
struct tcf_exts_miss_cookie_node {
const struct tcf_chain *chain;
const struct tcf_proto *tp;
const struct tcf_exts *exts;
u32 chain_index;
u32 tp_prio;
u32 handle;
u32 miss_cookie_base;
struct rcu_head rcu;
};
/* Each tc action entry cookie will be comprised of 32bit miss_cookie_base +
* action index in the exts tc actions array.
*/
union tcf_exts_miss_cookie {
struct {
u32 miss_cookie_base;
u32 act_index;
};
u64 miss_cookie;
};
#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
static int
tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp,
u32 handle)
{
struct tcf_exts_miss_cookie_node *n;
static u32 next;
int err;
if (WARN_ON(!handle || !tp->ops->get_exts))
return -EINVAL;
n = kzalloc(sizeof(*n), GFP_KERNEL);
if (!n)
return -ENOMEM;
n->chain_index = tp->chain->index;
n->chain = tp->chain;
n->tp_prio = tp->prio;
n->tp = tp;
n->exts = exts;
n->handle = handle;
err = xa_alloc_cyclic(&tcf_exts_miss_cookies_xa, &n->miss_cookie_base,
n, xa_limit_32b, &next, GFP_KERNEL);
if (err)
goto err_xa_alloc;
exts->miss_cookie_node = n;
return 0;
err_xa_alloc:
kfree(n);
return err;
}
static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts)
{
struct tcf_exts_miss_cookie_node *n;
if (!exts->miss_cookie_node)
return;
n = exts->miss_cookie_node;
xa_erase(&tcf_exts_miss_cookies_xa, n->miss_cookie_base);
kfree_rcu(n, rcu);
}
static struct tcf_exts_miss_cookie_node *
tcf_exts_miss_cookie_lookup(u64 miss_cookie, int *act_index)
{
union tcf_exts_miss_cookie mc = { .miss_cookie = miss_cookie, };
*act_index = mc.act_index;
return xa_load(&tcf_exts_miss_cookies_xa, mc.miss_cookie_base);
}
#else /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */
static int
tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp,
u32 handle)
{
return 0;
}
static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts)
{
}
#endif /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */
static u64 tcf_exts_miss_cookie_get(u32 miss_cookie_base, int act_index)
{
union tcf_exts_miss_cookie mc = { .act_index = act_index, };
if (!miss_cookie_base)
return 0;
mc.miss_cookie_base = miss_cookie_base;
return mc.miss_cookie;
}
#ifdef CONFIG_NET_CLS_ACT #ifdef CONFIG_NET_CLS_ACT
DEFINE_STATIC_KEY_FALSE(tc_skb_ext_tc); DEFINE_STATIC_KEY_FALSE(tc_skb_ext_tc);
EXPORT_SYMBOL(tc_skb_ext_tc); EXPORT_SYMBOL(tc_skb_ext_tc);
...@@ -1549,6 +1653,8 @@ static inline int __tcf_classify(struct sk_buff *skb, ...@@ -1549,6 +1653,8 @@ static inline int __tcf_classify(struct sk_buff *skb,
const struct tcf_proto *orig_tp, const struct tcf_proto *orig_tp,
struct tcf_result *res, struct tcf_result *res,
bool compat_mode, bool compat_mode,
struct tcf_exts_miss_cookie_node *n,
int act_index,
u32 *last_executed_chain) u32 *last_executed_chain)
{ {
#ifdef CONFIG_NET_CLS_ACT #ifdef CONFIG_NET_CLS_ACT
...@@ -1560,13 +1666,36 @@ static inline int __tcf_classify(struct sk_buff *skb, ...@@ -1560,13 +1666,36 @@ static inline int __tcf_classify(struct sk_buff *skb,
#endif #endif
for (; tp; tp = rcu_dereference_bh(tp->next)) { for (; tp; tp = rcu_dereference_bh(tp->next)) {
__be16 protocol = skb_protocol(skb, false); __be16 protocol = skb_protocol(skb, false);
int err; int err = 0;
if (n) {
struct tcf_exts *exts;
if (n->tp_prio != tp->prio)
continue;
/* We re-lookup the tp and chain based on index instead
* of having hard refs and locks to them, so do a sanity
* check if any of tp,chain,exts was replaced by the
* time we got here with a cookie from hardware.
*/
if (unlikely(n->tp != tp || n->tp->chain != n->chain ||
!tp->ops->get_exts))
return TC_ACT_SHOT;
exts = tp->ops->get_exts(tp, n->handle);
if (unlikely(!exts || n->exts != exts))
return TC_ACT_SHOT;
n = NULL;
err = tcf_exts_exec_ex(skb, exts, act_index, res);
} else {
if (tp->protocol != protocol && if (tp->protocol != protocol &&
tp->protocol != htons(ETH_P_ALL)) tp->protocol != htons(ETH_P_ALL))
continue; continue;
err = tc_classify(skb, tp, res); err = tc_classify(skb, tp, res);
}
#ifdef CONFIG_NET_CLS_ACT #ifdef CONFIG_NET_CLS_ACT
if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) { if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
first_tp = orig_tp; first_tp = orig_tp;
...@@ -1582,6 +1711,9 @@ static inline int __tcf_classify(struct sk_buff *skb, ...@@ -1582,6 +1711,9 @@ static inline int __tcf_classify(struct sk_buff *skb,
return err; return err;
} }
if (unlikely(n))
return TC_ACT_SHOT;
return TC_ACT_UNSPEC; /* signal: continue lookup */ return TC_ACT_UNSPEC; /* signal: continue lookup */
#ifdef CONFIG_NET_CLS_ACT #ifdef CONFIG_NET_CLS_ACT
reset: reset:
...@@ -1606,21 +1738,35 @@ int tcf_classify(struct sk_buff *skb, ...@@ -1606,21 +1738,35 @@ int tcf_classify(struct sk_buff *skb,
#if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT) #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
u32 last_executed_chain = 0; u32 last_executed_chain = 0;
return __tcf_classify(skb, tp, tp, res, compat_mode, return __tcf_classify(skb, tp, tp, res, compat_mode, NULL, 0,
&last_executed_chain); &last_executed_chain);
#else #else
u32 last_executed_chain = tp ? tp->chain->index : 0; u32 last_executed_chain = tp ? tp->chain->index : 0;
struct tcf_exts_miss_cookie_node *n = NULL;
const struct tcf_proto *orig_tp = tp; const struct tcf_proto *orig_tp = tp;
struct tc_skb_ext *ext; struct tc_skb_ext *ext;
int act_index = 0;
int ret; int ret;
if (block) { if (block) {
ext = skb_ext_find(skb, TC_SKB_EXT); ext = skb_ext_find(skb, TC_SKB_EXT);
if (ext && ext->chain) { if (ext && (ext->chain || ext->act_miss)) {
struct tcf_chain *fchain; struct tcf_chain *fchain;
u32 chain;
fchain = tcf_chain_lookup_rcu(block, ext->chain); if (ext->act_miss) {
n = tcf_exts_miss_cookie_lookup(ext->act_miss_cookie,
&act_index);
if (!n)
return TC_ACT_SHOT;
chain = n->chain_index;
} else {
chain = ext->chain;
}
fchain = tcf_chain_lookup_rcu(block, chain);
if (!fchain) if (!fchain)
return TC_ACT_SHOT; return TC_ACT_SHOT;
...@@ -1632,7 +1778,7 @@ int tcf_classify(struct sk_buff *skb, ...@@ -1632,7 +1778,7 @@ int tcf_classify(struct sk_buff *skb,
} }
} }
ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode, ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode, n, act_index,
&last_executed_chain); &last_executed_chain);
if (tc_skb_ext_tc_enabled()) { if (tc_skb_ext_tc_enabled()) {
...@@ -3056,9 +3202,48 @@ static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb) ...@@ -3056,9 +3202,48 @@ static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
return skb->len; return skb->len;
} }
int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action,
int police, struct tcf_proto *tp, u32 handle,
bool use_action_miss)
{
int err = 0;
#ifdef CONFIG_NET_CLS_ACT
exts->type = 0;
exts->nr_actions = 0;
/* Note: we do not own yet a reference on net.
* This reference might be taken later from tcf_exts_get_net().
*/
exts->net = net;
exts->actions = kcalloc(TCA_ACT_MAX_PRIO, sizeof(struct tc_action *),
GFP_KERNEL);
if (!exts->actions)
return -ENOMEM;
#endif
exts->action = action;
exts->police = police;
if (!use_action_miss)
return 0;
err = tcf_exts_miss_cookie_base_alloc(exts, tp, handle);
if (err)
goto err_miss_alloc;
return 0;
err_miss_alloc:
tcf_exts_destroy(exts);
return err;
}
EXPORT_SYMBOL(tcf_exts_init_ex);
void tcf_exts_destroy(struct tcf_exts *exts) void tcf_exts_destroy(struct tcf_exts *exts)
{ {
#ifdef CONFIG_NET_CLS_ACT #ifdef CONFIG_NET_CLS_ACT
tcf_exts_miss_cookie_base_destroy(exts);
if (exts->actions) { if (exts->actions) {
tcf_action_destroy(exts->actions, TCA_ACT_UNBIND); tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
kfree(exts->actions); kfree(exts->actions);
...@@ -3547,6 +3732,7 @@ static int tc_setup_offload_act(struct tc_action *act, ...@@ -3547,6 +3732,7 @@ static int tc_setup_offload_act(struct tc_action *act,
int tc_setup_action(struct flow_action *flow_action, int tc_setup_action(struct flow_action *flow_action,
struct tc_action *actions[], struct tc_action *actions[],
u32 miss_cookie_base,
struct netlink_ext_ack *extack) struct netlink_ext_ack *extack)
{ {
int i, j, k, index, err = 0; int i, j, k, index, err = 0;
...@@ -3578,6 +3764,8 @@ int tc_setup_action(struct flow_action *flow_action, ...@@ -3578,6 +3764,8 @@ int tc_setup_action(struct flow_action *flow_action,
entry[k].hw_stats = tc_act_hw_stats(act->hw_stats); entry[k].hw_stats = tc_act_hw_stats(act->hw_stats);
entry[k].hw_index = act->tcfa_index; entry[k].hw_index = act->tcfa_index;
entry[k].cookie = (unsigned long)act; entry[k].cookie = (unsigned long)act;
entry[k].miss_cookie =
tcf_exts_miss_cookie_get(miss_cookie_base, i);
} }
j += index; j += index;
...@@ -3600,10 +3788,15 @@ int tc_setup_offload_action(struct flow_action *flow_action, ...@@ -3600,10 +3788,15 @@ int tc_setup_offload_action(struct flow_action *flow_action,
struct netlink_ext_ack *extack) struct netlink_ext_ack *extack)
{ {
#ifdef CONFIG_NET_CLS_ACT #ifdef CONFIG_NET_CLS_ACT
u32 miss_cookie_base;
if (!exts) if (!exts)
return 0; return 0;
return tc_setup_action(flow_action, exts->actions, extack); miss_cookie_base = exts->miss_cookie_node ?
exts->miss_cookie_node->miss_cookie_base : 0;
return tc_setup_action(flow_action, exts->actions, miss_cookie_base,
extack);
#else #else
return 0; return 0;
#endif #endif
...@@ -3771,6 +3964,8 @@ static int __init tc_filter_init(void) ...@@ -3771,6 +3964,8 @@ static int __init tc_filter_init(void)
if (err) if (err)
goto err_register_pernet_subsys; goto err_register_pernet_subsys;
xa_init_flags(&tcf_exts_miss_cookies_xa, XA_FLAGS_ALLOC1);
rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL, rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL,
RTNL_FLAG_DOIT_UNLOCKED); RTNL_FLAG_DOIT_UNLOCKED);
rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL, rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment