net: call rcu_read_lock early in process_backlog

[ Upstream commit 2c17d27c36dcce2b6bf689f41a46b9e909877c21 ]

Incoming packet should be either in backlog queue or
in RCU read-side section. Otherwise, the final sequence of
flush_backlog() and synchronize_net() may miss packets
that can run without device reference:

CPU 1                  CPU 2
                       skb->dev: no reference
                       process_backlog:__skb_dequeue
                       process_backlog:local_irq_enable

on_each_cpu for
flush_backlog =>       IPI(hardirq): flush_backlog
                       - packet not found in backlog

                       CPU delayed ...
synchronize_net
- no ongoing RCU
read-side sections

netdev_run_todo,
rcu_barrier: no
ongoing callbacks
                       __netif_receive_skb_core:rcu_read_lock
                       - too late
free dev
                       process packet for freed dev

Fixes: 6e583ce5242f ("net: eliminate refcounting in backlog queue")
Cc: Eric W. Biederman <ebiederm@xmission.com>
Cc: Stephen Hemminger <stephen@networkplumber.org>
Signed-off-by: Julian Anastasov <ja@ssi.bg>
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Jiri Slaby <jslaby@suse.cz>

diff --git a/net/core/dev.c b/net/core/dev.c
index 89c6134a979d075ddadec461e0ad91dd4fd28089..f991f5d3371d3eae80f5634703e02dbf3bb6f7e4 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3521,8 +3521,6 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
 
        pt_prev = NULL;
 
-       rcu_read_lock();
-
 another_round:
        skb->skb_iif = skb->dev->ifindex;
 
@@ -3532,7 +3530,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
            skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
                skb = skb_vlan_untag(skb);
                if (unlikely(!skb))
-                       goto unlock;
+                       goto out;
        }
 
 #ifdef CONFIG_NET_CLS_ACT
@@ -3557,7 +3555,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
 #ifdef CONFIG_NET_CLS_ACT
        skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
        if (!skb)
-               goto unlock;
+               goto out;
 ncls:
 #endif
 
@@ -3572,7 +3570,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
                if (vlan_do_receive(&skb))
                        goto another_round;
                else if (unlikely(!skb))
-                       goto unlock;
+                       goto out;
        }
 
        rx_handler = rcu_dereference(skb->dev->rx_handler);
@@ -3584,7 +3582,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
                switch (rx_handler(&skb)) {
                case RX_HANDLER_CONSUMED:
                        ret = NET_RX_SUCCESS;
-                       goto unlock;
+                       goto out;
                case RX_HANDLER_ANOTHER:
                        goto another_round;
                case RX_HANDLER_EXACT:
@@ -3636,8 +3634,6 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
                ret = NET_RX_DROP;
        }
 
-unlock:
-       rcu_read_unlock();
 out:
        return ret;
 }
@@ -3684,29 +3680,30 @@ static int __netif_receive_skb(struct sk_buff *skb)
  */
 int netif_receive_skb(struct sk_buff *skb)
 {
+       int ret;
+
        net_timestamp_check(netdev_tstamp_prequeue, skb);
 
        if (skb_defer_rx_timestamp(skb))
                return NET_RX_SUCCESS;
 
+       rcu_read_lock();
+
 #ifdef CONFIG_RPS
        if (static_key_false(&rps_needed)) {
                struct rps_dev_flow voidflow, *rflow = &voidflow;
-               int cpu, ret;
-
-               rcu_read_lock();
-
-               cpu = get_rps_cpu(skb->dev, skb, &rflow);
+               int cpu = get_rps_cpu(skb->dev, skb, &rflow);
 
                if (cpu >= 0) {
                        ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
                        rcu_read_unlock();
                        return ret;
                }
-               rcu_read_unlock();
        }
 #endif
-       return __netif_receive_skb(skb);
+       ret = __netif_receive_skb(skb);
+       rcu_read_unlock();
+       return ret;
 }
 EXPORT_SYMBOL(netif_receive_skb);
 
@@ -4116,8 +4113,10 @@ static int process_backlog(struct napi_struct *napi, int quota)
                unsigned int qlen;
 
                while ((skb = __skb_dequeue(&sd->process_queue))) {
+                       rcu_read_lock();
                        local_irq_enable();
                        __netif_receive_skb(skb);
+                       rcu_read_unlock();
                        local_irq_disable();
                        input_queue_head_incr(sd);
                        if (++work >= quota) {