rfs: annotate lockless accesses to sk->sk_rxhash

[ Upstream commit 1e5c647c3f6d4f8497dedcd226204e1880e0ffb3 ]

Add READ_ONCE()/WRITE_ONCE() on accesses to sk->sk_rxhash.

This also prevents a (smart ?) compiler to remove the condition in:

if (sk->sk_rxhash != newval)
	sk->sk_rxhash = newval;

We need the condition to avoid dirtying a shared cache line.

Fixes: fec5e652e58f ("rfs: Receive Flow Steering")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Simon Horman <simon.horman@corigine.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/include/net/sock.h b/include/net/sock.h
index 3da0601b573ed10bf48c1bfc77c7506cc8aa29bd..51b499d7454997a319608da79cb033c9b7797161 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -1073,8 +1073,12 @@ static inline void sock_rps_record_flow(const struct sock *sk)
                 * OR   an additional socket flag
                 * [1] : sk_state and sk_prot are in the same cache line.
                 */
-               if (sk->sk_state == TCP_ESTABLISHED)
-                       sock_rps_record_flow_hash(sk->sk_rxhash);
+               if (sk->sk_state == TCP_ESTABLISHED) {
+                       /* This READ_ONCE() is paired with the WRITE_ONCE()
+                        * from sock_rps_save_rxhash() and sock_rps_reset_rxhash().
+                        */
+                       sock_rps_record_flow_hash(READ_ONCE(sk->sk_rxhash));
+               }
        }
 #endif
 }
@@ -1083,15 +1087,19 @@ static inline void sock_rps_save_rxhash(struct sock *sk,
                                        const struct sk_buff *skb)
 {
 #ifdef CONFIG_RPS
-       if (unlikely(sk->sk_rxhash != skb->hash))
-               sk->sk_rxhash = skb->hash;
+       /* The following WRITE_ONCE() is paired with the READ_ONCE()
+        * here, and another one in sock_rps_record_flow().
+        */
+       if (unlikely(READ_ONCE(sk->sk_rxhash) != skb->hash))
+               WRITE_ONCE(sk->sk_rxhash, skb->hash);
 #endif
 }
 
 static inline void sock_rps_reset_rxhash(struct sock *sk)
 {
 #ifdef CONFIG_RPS
-       sk->sk_rxhash = 0;
+       /* Paired with READ_ONCE() in sock_rps_record_flow() */
+       WRITE_ONCE(sk->sk_rxhash, 0);
 #endif
 }