Skip to content

Commit

Permalink
neighbour: switch to standard rcu, instead of rcu_bh
Browse files Browse the repository at this point in the history 
rcu_bh is no longer a win, especially for objects freed
with standard call_rcu().

Switch neighbour code to no longer disable BH when not necessary.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
  • Loading branch information
@kuba-moo
Eric Dumazet authored and kuba-moo committed Mar 22, 2023
1 parent 4c5c496 commit 09eed11
Show file tree
Hide file tree
Showing 13 changed files with 87 additions and 83 deletions.
8 changes: 4 additions & 4 deletions include/net/arp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,11 @@ static inline struct neighbour *__ipv4_neigh_lookup(struct net_device *dev, u32
{
struct neighbour *n;

rcu_read_lock_bh();
rcu_read_lock();
n = __ipv4_neigh_lookup_noref(dev, key);
if (n && !refcount_inc_not_zero(&n->refcnt))
n = NULL;
rcu_read_unlock_bh();
rcu_read_unlock();

return n;
}
Expand All @@ -51,10 +51,10 @@ static inline void __ipv4_confirm_neigh(struct net_device *dev, u32 key)
{
struct neighbour *n;

rcu_read_lock_bh();
rcu_read_lock();
n = __ipv4_neigh_lookup_noref(dev, key);
neigh_confirm(n);
rcu_read_unlock_bh();
rcu_read_unlock();
}

void arp_init(void);
Expand Down
12 changes: 6 additions & 6 deletions include/net/ndisc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,11 +395,11 @@ static inline struct neighbour *__ipv6_neigh_lookup(struct net_device *dev, cons
{
struct neighbour *n;

rcu_read_lock_bh();
rcu_read_lock();
n = __ipv6_neigh_lookup_noref(dev, pkey);
if (n && !refcount_inc_not_zero(&n->refcnt))
n = NULL;
rcu_read_unlock_bh();
rcu_read_unlock();

return n;
}
Expand All @@ -409,21 +409,21 @@ static inline void __ipv6_confirm_neigh(struct net_device *dev,
{
struct neighbour *n;

rcu_read_lock_bh();
rcu_read_lock();
n = __ipv6_neigh_lookup_noref(dev, pkey);
neigh_confirm(n);
rcu_read_unlock_bh();
rcu_read_unlock();
}

static inline void __ipv6_confirm_neigh_stub(struct net_device *dev,
const void *pkey)
{
struct neighbour *n;

rcu_read_lock_bh();
rcu_read_lock();
n = __ipv6_neigh_lookup_noref_stub(dev, pkey);
neigh_confirm(n);
rcu_read_unlock_bh();
rcu_read_unlock();
}

/* uses ipv6_stub and is meant for use outside of IPv6 core */
Expand Down
6 changes: 3 additions & 3 deletions include/net/neighbour.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,14 +299,14 @@ static inline struct neighbour *___neigh_lookup_noref(
const void *pkey,
struct net_device *dev)
{
struct neigh_hash_table *nht = rcu_dereference_bh(tbl->nht);
struct neigh_hash_table *nht = rcu_dereference(tbl->nht);
struct neighbour *n;
u32 hash_val;

hash_val = hash(pkey, dev, nht->hash_rnd) >> (32 - nht->hash_shift);
for (n = rcu_dereference_bh(nht->hash_buckets[hash_val]);
for (n = rcu_dereference(nht->hash_buckets[hash_val]);
n != NULL;
n = rcu_dereference_bh(n->next)) {
n = rcu_dereference(n->next)) {
if (n->dev == dev && key_eq(n, pkey))
return n;
}
Expand Down
6 changes: 3 additions & 3 deletions include/net/nexthop.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ static inline struct fib6_nh *nexthop_fib6_nh(struct nexthop *nh)
}

/* Variant of nexthop_fib6_nh().
* Caller should either hold rcu_read_lock_bh(), or RTNL.
* Caller should either hold rcu_read_lock(), or RTNL.
*/
static inline struct fib6_nh *nexthop_fib6_nh_bh(struct nexthop *nh)
{
Expand All @@ -507,13 +507,13 @@ static inline struct fib6_nh *nexthop_fib6_nh_bh(struct nexthop *nh)
if (nh->is_group) {
struct nh_group *nh_grp;

nh_grp = rcu_dereference_bh_rtnl(nh->nh_grp);
nh_grp = rcu_dereference_rtnl(nh->nh_grp);
nh = nexthop_mpath_select(nh_grp, 0);
if (!nh)
return NULL;
}

nhi = rcu_dereference_bh_rtnl(nh->nh_info);
nhi = rcu_dereference_rtnl(nh->nh_info);
if (nhi->family == AF_INET6)
return &nhi->fib6_nh;

Expand Down
16 changes: 10 additions & 6 deletions net/core/filter.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2204,7 +2204,7 @@ static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb,
return -ENOMEM;
}

rcu_read_lock_bh();
rcu_read_lock();
if (!nh) {
dst = skb_dst(skb);
nexthop = rt6_nexthop(container_of(dst, struct rt6_info, dst),
Expand All @@ -2217,10 +2217,12 @@ static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb,
int ret;

sock_confirm_neigh(skb, neigh);
local_bh_disable();
dev_xmit_recursion_inc();
ret = neigh_output(neigh, skb, false);
dev_xmit_recursion_dec();
rcu_read_unlock_bh();
local_bh_enable();
rcu_read_unlock();
return ret;
}
rcu_read_unlock_bh();
Expand Down Expand Up @@ -2302,7 +2304,7 @@ static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb,
return -ENOMEM;
}

rcu_read_lock_bh();
rcu_read_lock();
if (!nh) {
struct dst_entry *dst = skb_dst(skb);
struct rtable *rt = container_of(dst, struct rtable, dst);
Expand All @@ -2314,21 +2316,23 @@ static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb,
} else if (nh->nh_family == AF_INET) {
neigh = ip_neigh_gw4(dev, nh->ipv4_nh);
} else {
rcu_read_unlock_bh();
rcu_read_unlock();
goto out_drop;
}

if (likely(!IS_ERR(neigh))) {
int ret;

sock_confirm_neigh(skb, neigh);
local_bh_disable();
dev_xmit_recursion_inc();
ret = neigh_output(neigh, skb, is_v6gw);
dev_xmit_recursion_dec();
rcu_read_unlock_bh();
local_bh_enable();
rcu_read_unlock();
return ret;
}
rcu_read_unlock_bh();
rcu_read_unlock();
out_drop:
kfree_skb(skb);
return -ENETDOWN;
Expand Down
64 changes: 32 additions & 32 deletions net/core/neighbour.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -614,15 +614,15 @@ struct neighbour *neigh_lookup(struct neigh_table *tbl, const void *pkey,

NEIGH_CACHE_STAT_INC(tbl, lookups);

rcu_read_lock_bh();
rcu_read_lock();
n = __neigh_lookup_noref(tbl, pkey, dev);
if (n) {
if (!refcount_inc_not_zero(&n->refcnt))
n = NULL;
NEIGH_CACHE_STAT_INC(tbl, hits);
}

rcu_read_unlock_bh();
rcu_read_unlock();
return n;
}
EXPORT_SYMBOL(neigh_lookup);
Expand Down Expand Up @@ -2184,11 +2184,11 @@ static int neightbl_fill_info(struct sk_buff *skb, struct neigh_table *tbl,
.ndtc_proxy_qlen = tbl->proxy_queue.qlen,
};

rcu_read_lock_bh();
nht = rcu_dereference_bh(tbl->nht);
rcu_read_lock();
nht = rcu_dereference(tbl->nht);
ndc.ndtc_hash_rnd = nht->hash_rnd[0];
ndc.ndtc_hash_mask = ((1 << nht->hash_shift) - 1);
rcu_read_unlock_bh();
rcu_read_unlock();

if (nla_put(skb, NDTA_CONFIG, sizeof(ndc), &ndc))
goto nla_put_failure;
Expand Down Expand Up @@ -2703,15 +2703,15 @@ static int neigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
if (filter->dev_idx || filter->master_idx)
flags |= NLM_F_DUMP_FILTERED;

rcu_read_lock_bh();
nht = rcu_dereference_bh(tbl->nht);
rcu_read_lock();
nht = rcu_dereference(tbl->nht);

for (h = s_h; h < (1 << nht->hash_shift); h++) {
if (h > s_h)
s_idx = 0;
for (n = rcu_dereference_bh(nht->hash_buckets[h]), idx = 0;
for (n = rcu_dereference(nht->hash_buckets[h]), idx = 0;
n != NULL;
n = rcu_dereference_bh(n->next)) {
n = rcu_dereference(n->next)) {
if (idx < s_idx || !net_eq(dev_net(n->dev), net))
goto next;
if (neigh_ifindex_filtered(n->dev, filter->dev_idx) ||
Expand All @@ -2730,7 +2730,7 @@ static int neigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
}
rc = skb->len;
out:
rcu_read_unlock_bh();
rcu_read_unlock();
cb->args[1] = h;
cb->args[2] = idx;
return rc;
Expand Down Expand Up @@ -3075,20 +3075,20 @@ void neigh_for_each(struct neigh_table *tbl, void (*cb)(struct neighbour *, void
int chain;
struct neigh_hash_table *nht;

rcu_read_lock_bh();
nht = rcu_dereference_bh(tbl->nht);
rcu_read_lock();
nht = rcu_dereference(tbl->nht);

read_lock(&tbl->lock); /* avoid resizes */
read_lock_bh(&tbl->lock); /* avoid resizes */
for (chain = 0; chain < (1 << nht->hash_shift); chain++) {
struct neighbour *n;

for (n = rcu_dereference_bh(nht->hash_buckets[chain]);
for (n = rcu_dereference(nht->hash_buckets[chain]);
n != NULL;
n = rcu_dereference_bh(n->next))
n = rcu_dereference(n->next))
cb(n, cookie);
}
read_unlock(&tbl->lock);
rcu_read_unlock_bh();
read_unlock_bh(&tbl->lock);
rcu_read_unlock();
}
EXPORT_SYMBOL(neigh_for_each);

Expand Down Expand Up @@ -3138,7 +3138,7 @@ int neigh_xmit(int index, struct net_device *dev,
tbl = neigh_tables[index];
if (!tbl)
goto out;
rcu_read_lock_bh();
rcu_read_lock();
if (index == NEIGH_ARP_TABLE) {
u32 key = *((u32 *)addr);

Expand All @@ -3150,11 +3150,11 @@ int neigh_xmit(int index, struct net_device *dev,
neigh = __neigh_create(tbl, addr, dev, false);
err = PTR_ERR(neigh);
if (IS_ERR(neigh)) {
rcu_read_unlock_bh();
rcu_read_unlock();
goto out_kfree_skb;
}
err = neigh->output(neigh, skb);
rcu_read_unlock_bh();
rcu_read_unlock();
}
else if (index == NEIGH_LINK_TABLE) {
err = dev_hard_header(skb, dev, ntohs(skb->protocol),
Expand Down Expand Up @@ -3183,7 +3183,7 @@ static struct neighbour *neigh_get_first(struct seq_file *seq)

state->flags &= ~NEIGH_SEQ_IS_PNEIGH;
for (bucket = 0; bucket < (1 << nht->hash_shift); bucket++) {
n = rcu_dereference_bh(nht->hash_buckets[bucket]);
n = rcu_dereference(nht->hash_buckets[bucket]);

while (n) {
if (!net_eq(dev_net(n->dev), net))
Expand All @@ -3201,7 +3201,7 @@ static struct neighbour *neigh_get_first(struct seq_file *seq)
if (READ_ONCE(n->nud_state) & ~NUD_NOARP)
break;
next:
n = rcu_dereference_bh(n->next);
n = rcu_dereference(n->next);
}

if (n)
Expand All @@ -3225,7 +3225,7 @@ static struct neighbour *neigh_get_next(struct seq_file *seq,
if (v)
return n;
}
n = rcu_dereference_bh(n->next);
n = rcu_dereference(n->next);

while (1) {
while (n) {
Expand All @@ -3243,7 +3243,7 @@ static struct neighbour *neigh_get_next(struct seq_file *seq,
if (READ_ONCE(n->nud_state) & ~NUD_NOARP)
break;
next:
n = rcu_dereference_bh(n->next);
n = rcu_dereference(n->next);
}

if (n)
Expand All @@ -3252,7 +3252,7 @@ static struct neighbour *neigh_get_next(struct seq_file *seq,
if (++state->bucket >= (1 << nht->hash_shift))
break;

n = rcu_dereference_bh(nht->hash_buckets[state->bucket]);
n = rcu_dereference(nht->hash_buckets[state->bucket]);
}

if (n && pos)
Expand Down Expand Up @@ -3354,17 +3354,17 @@ static void *neigh_get_idx_any(struct seq_file *seq, loff_t *pos)

void *neigh_seq_start(struct seq_file *seq, loff_t *pos, struct neigh_table *tbl, unsigned int neigh_seq_flags)
__acquires(tbl->lock)
__acquires(rcu_bh)
__acquires(rcu)
{
struct neigh_seq_state *state = seq->private;

state->tbl = tbl;
state->bucket = 0;
state->flags = (neigh_seq_flags & ~NEIGH_SEQ_IS_PNEIGH);

rcu_read_lock_bh();
state->nht = rcu_dereference_bh(tbl->nht);
read_lock(&tbl->lock);
rcu_read_lock();
state->nht = rcu_dereference(tbl->nht);
read_lock_bh(&tbl->lock);

return *pos ? neigh_get_idx_any(seq, pos) : SEQ_START_TOKEN;
}
Expand Down Expand Up @@ -3399,13 +3399,13 @@ EXPORT_SYMBOL(neigh_seq_next);

void neigh_seq_stop(struct seq_file *seq, void *v)
__releases(tbl->lock)
__releases(rcu_bh)
__releases(rcu)
{
struct neigh_seq_state *state = seq->private;
struct neigh_table *tbl = state->tbl;

read_unlock(&tbl->lock);
rcu_read_unlock_bh();
read_unlock_bh(&tbl->lock);
rcu_read_unlock();
}
EXPORT_SYMBOL(neigh_seq_stop);

Expand Down
4 changes: 2 additions & 2 deletions net/ipv4/fib_semantics.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2191,7 +2191,7 @@ static bool fib_good_nh(const struct fib_nh *nh)
if (nh->fib_nh_scope == RT_SCOPE_LINK) {
struct neighbour *n;

rcu_read_lock_bh();
rcu_read_lock();

if (likely(nh->fib_nh_gw_family == AF_INET))
n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
Expand All @@ -2204,7 +2204,7 @@ static bool fib_good_nh(const struct fib_nh *nh)
if (n)
state = READ_ONCE(n->nud_state);

rcu_read_unlock_bh();
rcu_read_unlock();
}

return !!(state & NUD_VALID);
Expand Down
Loading

0 comments on commit 09eed11

Please sign in to comment.