net/mlx5: DR, Cache STE shadow memory

During rule insertion on each ICM memory chunk we also allocate shadow memory
used for management. This includes the hw_ste, dr_ste and miss list per entry.
Since the scale of these allocations is large we noticed a performance hiccup
that happens once malloc and free are stressed.
In extreme usecases when ~1M chunks are freed at once, it might take up to 40
seconds to complete this, up to the point the kernel sees this as self-detected
stall on CPU:

 rcu: INFO: rcu_sched self-detected stall on CPU

To resolve this we will increase the reuse of shadow memory.
Doing this we see that a time in the aforementioned usecase dropped from ~40
seconds to ~8-10 seconds.

Fixes: 29cf8febd185 ("net/mlx5: DR, ICM pool memory allocator")
Signed-off-by: Alex Vesker <valex@nvidia.com>
Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c
index 7f6fd9c5e371b4af30b9cc78307c514fbf68a808..f496b7e9401b19d021f8efa60d8ae2516d05cd1c 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c
@@ -136,37 +136,35 @@ static void dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr *icm_mr)
        kvfree(icm_mr);
 }
 
-static int dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk)
+static int dr_icm_buddy_get_ste_size(struct mlx5dr_icm_buddy_mem *buddy)
 {
-       chunk->ste_arr = kvzalloc(chunk->num_of_entries *
-                                 sizeof(chunk->ste_arr[0]), GFP_KERNEL);
-       if (!chunk->ste_arr)
-               return -ENOMEM;
-
-       chunk->hw_ste_arr = kvzalloc(chunk->num_of_entries *
-                                    DR_STE_SIZE_REDUCED, GFP_KERNEL);
-       if (!chunk->hw_ste_arr)
-               goto out_free_ste_arr;
-
-       chunk->miss_list = kvmalloc(chunk->num_of_entries *
-                                   sizeof(chunk->miss_list[0]), GFP_KERNEL);
-       if (!chunk->miss_list)
-               goto out_free_hw_ste_arr;
+       /* We support only one type of STE size, both for ConnectX-5 and later
+        * devices. Once the support for match STE which has a larger tag is
+        * added (32B instead of 16B), the STE size for devices later than
+        * ConnectX-5 needs to account for that.
+        */
+       return DR_STE_SIZE_REDUCED;
+}
 
-       return 0;
+static void dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk, int offset)
+{
+       struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem;
+       int index = offset / DR_STE_SIZE;
 
-out_free_hw_ste_arr:
-       kvfree(chunk->hw_ste_arr);
-out_free_ste_arr:
-       kvfree(chunk->ste_arr);
-       return -ENOMEM;
+       chunk->ste_arr = &buddy->ste_arr[index];
+       chunk->miss_list = &buddy->miss_list[index];
+       chunk->hw_ste_arr = buddy->hw_ste_arr +
+                           index * dr_icm_buddy_get_ste_size(buddy);
 }
 
 static void dr_icm_chunk_ste_cleanup(struct mlx5dr_icm_chunk *chunk)
 {
-       kvfree(chunk->miss_list);
-       kvfree(chunk->hw_ste_arr);
-       kvfree(chunk->ste_arr);
+       struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem;
+
+       memset(chunk->hw_ste_arr, 0,
+              chunk->num_of_entries * dr_icm_buddy_get_ste_size(buddy));
+       memset(chunk->ste_arr, 0,
+              chunk->num_of_entries * sizeof(chunk->ste_arr[0]));
 }
 
 static enum mlx5dr_icm_type
@@ -189,6 +187,44 @@ static void dr_icm_chunk_destroy(struct mlx5dr_icm_chunk *chunk,
        kvfree(chunk);
 }
 
+static int dr_icm_buddy_init_ste_cache(struct mlx5dr_icm_buddy_mem *buddy)
+{
+       int num_of_entries =
+               mlx5dr_icm_pool_chunk_size_to_entries(buddy->pool->max_log_chunk_sz);
+
+       buddy->ste_arr = kvcalloc(num_of_entries,
+                                 sizeof(struct mlx5dr_ste), GFP_KERNEL);
+       if (!buddy->ste_arr)
+               return -ENOMEM;
+
+       /* Preallocate full STE size on non-ConnectX-5 devices since
+        * we need to support both full and reduced with the same cache.
+        */
+       buddy->hw_ste_arr = kvcalloc(num_of_entries,
+                                    dr_icm_buddy_get_ste_size(buddy), GFP_KERNEL);
+       if (!buddy->hw_ste_arr)
+               goto free_ste_arr;
+
+       buddy->miss_list = kvmalloc(num_of_entries * sizeof(struct list_head), GFP_KERNEL);
+       if (!buddy->miss_list)
+               goto free_hw_ste_arr;
+
+       return 0;
+
+free_hw_ste_arr:
+       kvfree(buddy->hw_ste_arr);
+free_ste_arr:
+       kvfree(buddy->ste_arr);
+       return -ENOMEM;
+}
+
+static void dr_icm_buddy_cleanup_ste_cache(struct mlx5dr_icm_buddy_mem *buddy)
+{
+       kvfree(buddy->ste_arr);
+       kvfree(buddy->hw_ste_arr);
+       kvfree(buddy->miss_list);
+}
+
 static int dr_icm_buddy_create(struct mlx5dr_icm_pool *pool)
 {
        struct mlx5dr_icm_buddy_mem *buddy;
@@ -208,11 +244,19 @@ static int dr_icm_buddy_create(struct mlx5dr_icm_pool *pool)
        buddy->icm_mr = icm_mr;
        buddy->pool = pool;
 
+       if (pool->icm_type == DR_ICM_TYPE_STE) {
+               /* Reduce allocations by preallocating and reusing the STE structures */
+               if (dr_icm_buddy_init_ste_cache(buddy))
+                       goto err_cleanup_buddy;
+       }
+
        /* add it to the -start- of the list in order to search in it first */
        list_add(&buddy->list_node, &pool->buddy_mem_list);
 
        return 0;
 
+err_cleanup_buddy:
+       mlx5dr_buddy_cleanup(buddy);
 err_free_buddy:
        kvfree(buddy);
 free_mr:
@@ -234,6 +278,9 @@ static void dr_icm_buddy_destroy(struct mlx5dr_icm_buddy_mem *buddy)
 
        mlx5dr_buddy_cleanup(buddy);
 
+       if (buddy->pool->icm_type == DR_ICM_TYPE_STE)
+               dr_icm_buddy_cleanup_ste_cache(buddy);
+
        kvfree(buddy);
 }
 
@@ -261,26 +308,18 @@ dr_icm_chunk_create(struct mlx5dr_icm_pool *pool,
        chunk->byte_size =
                mlx5dr_icm_pool_chunk_size_to_byte(chunk_size, pool->icm_type);
        chunk->seg = seg;
+       chunk->buddy_mem = buddy_mem_pool;
 
-       if (pool->icm_type == DR_ICM_TYPE_STE && dr_icm_chunk_ste_init(chunk)) {
-               mlx5dr_err(pool->dmn,
-                          "Failed to init ste arrays (order: %d)\n",
-                          chunk_size);
-               goto out_free_chunk;
-       }
+       if (pool->icm_type == DR_ICM_TYPE_STE)
+               dr_icm_chunk_ste_init(chunk, offset);
 
        buddy_mem_pool->used_memory += chunk->byte_size;
-       chunk->buddy_mem = buddy_mem_pool;
        INIT_LIST_HEAD(&chunk->chunk_list);
 
        /* chunk now is part of the used_list */
        list_add_tail(&chunk->chunk_list, &buddy_mem_pool->used_list);
 
        return chunk;
-
-out_free_chunk:
-       kvfree(chunk);
-       return NULL;
 }
 
 static bool dr_icm_pool_is_sync_required(struct mlx5dr_icm_pool *pool)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h
index c7c93131b762b64de9d11d2f71e44491f627788e..dfa223415fe24fed05e7dfd1ae728ca7f4d3b835 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h
@@ -160,6 +160,11 @@ struct mlx5dr_icm_buddy_mem {
         * sync_ste command sets them free.
         */
        struct list_head        hot_list;
+
+       /* Memory optimisation */
+       struct mlx5dr_ste       *ste_arr;
+       struct list_head        *miss_list;
+       u8                      *hw_ste_arr;
 };
 
 int mlx5dr_buddy_init(struct mlx5dr_icm_buddy_mem *buddy,