net/mlx5: Refactor LAG peer device lookout bus logic to mlx5 devcom

LAG peer device lookout bus logic required the usage of global lock,
mlx5_intf_mutex.
As part of the effort to remove this global lock, refactor LAG peer
device lookout to use mlx5 devcom layer.

Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Mark Bloch <mbloch@nvidia.com>
Reviewed-by: Jacob Keller <jacob.e.keller@intel.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
index 1fc03480c2ff25d1a0a5c1dcf403593b9921a4eb..6e3a8c22881f4d67966c10169ec421187a4f1c81 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
@@ -566,74 +566,6 @@ bool mlx5_same_hw_devs(struct mlx5_core_dev *dev, struct mlx5_core_dev *peer_dev
        return (fsystem_guid && psystem_guid && fsystem_guid == psystem_guid);
 }
 
-static u32 mlx5_gen_pci_id(const struct mlx5_core_dev *dev)
-{
-       return (u32)((pci_domain_nr(dev->pdev->bus) << 16) |
-                    (dev->pdev->bus->number << 8) |
-                    PCI_SLOT(dev->pdev->devfn));
-}
-
-static int _next_phys_dev(struct mlx5_core_dev *mdev,
-                         const struct mlx5_core_dev *curr)
-{
-       if (!mlx5_core_is_pf(mdev))
-               return 0;
-
-       if (mdev == curr)
-               return 0;
-
-       if (!mlx5_same_hw_devs(mdev, (struct mlx5_core_dev *)curr) &&
-           mlx5_gen_pci_id(mdev) != mlx5_gen_pci_id(curr))
-               return 0;
-
-       return 1;
-}
-
-static void *pci_get_other_drvdata(struct device *this, struct device *other)
-{
-       if (this->driver != other->driver)
-               return NULL;
-
-       return pci_get_drvdata(to_pci_dev(other));
-}
-
-static int next_phys_dev_lag(struct device *dev, const void *data)
-{
-       struct mlx5_core_dev *mdev, *this = (struct mlx5_core_dev *)data;
-
-       mdev = pci_get_other_drvdata(this->device, dev);
-       if (!mdev)
-               return 0;
-
-       if (!mlx5_lag_is_supported(mdev))
-               return 0;
-
-       return _next_phys_dev(mdev, data);
-}
-
-static struct mlx5_core_dev *mlx5_get_next_dev(struct mlx5_core_dev *dev,
-                                              int (*match)(struct device *dev, const void *data))
-{
-       struct device *next;
-
-       if (!mlx5_core_is_pf(dev))
-               return NULL;
-
-       next = bus_find_device(&pci_bus_type, NULL, dev, match);
-       if (!next)
-               return NULL;
-
-       put_device(next);
-       return pci_get_drvdata(to_pci_dev(next));
-}
-
-/* Must be called with intf_mutex held */
-struct mlx5_core_dev *mlx5_get_next_phys_dev_lag(struct mlx5_core_dev *dev)
-{
-       lockdep_assert_held(&mlx5_intf_mutex);
-       return mlx5_get_next_dev(dev, &next_phys_dev_lag);
-}
-
 void mlx5_dev_list_lock(void)
 {
        mutex_lock(&mlx5_intf_mutex);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
index af3fac090b828bd2acfe06a3b8ecb9f00e5db77d..f0b57f97739f0d8c1e5b8f19b3420ed5d34898f4 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
@@ -1212,13 +1212,14 @@ static void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev,
        dev->priv.lag = NULL;
 }
 
-/* Must be called with intf_mutex held */
+/* Must be called with HCA devcom component lock held */
 static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev)
 {
+       struct mlx5_devcom_comp_dev *pos = NULL;
        struct mlx5_lag *ldev = NULL;
        struct mlx5_core_dev *tmp_dev;
 
-       tmp_dev = mlx5_get_next_phys_dev_lag(dev);
+       tmp_dev = mlx5_devcom_get_next_peer_data(dev->priv.hca_devcom_comp, &pos);
        if (tmp_dev)
                ldev = mlx5_lag_dev(tmp_dev);
 
@@ -1275,10 +1276,13 @@ void mlx5_lag_add_mdev(struct mlx5_core_dev *dev)
        if (!mlx5_lag_is_supported(dev))
                return;
 
+       if (IS_ERR_OR_NULL(dev->priv.hca_devcom_comp))
+               return;
+
 recheck:
-       mlx5_dev_list_lock();
+       mlx5_devcom_comp_lock(dev->priv.hca_devcom_comp);
        err = __mlx5_lag_dev_add_mdev(dev);
-       mlx5_dev_list_unlock();
+       mlx5_devcom_comp_unlock(dev->priv.hca_devcom_comp);
 
        if (err) {
                msleep(100);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c
index 89ac3209277e16b6a562a008e93f36e6530cf968..f4d5c300ddd67ae07e9f94e36cd480c11f4fa655 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c
@@ -387,3 +387,17 @@ void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom,
        *pos = tmp;
        return data;
 }
+
+void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom)
+{
+       if (IS_ERR_OR_NULL(devcom))
+               return;
+       down_write(&devcom->comp->sem);
+}
+
+void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom)
+{
+       if (IS_ERR_OR_NULL(devcom))
+               return;
+       up_write(&devcom->comp->sem);
+}

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h
index 8220d180e33c96d9f751194e9e6826f08b7e6ba2..f06529cc3c61f9f6c26dfc44bfbea540bb3ab51d 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h
@@ -9,6 +9,7 @@
 enum mlx5_devcom_component {
        MLX5_DEVCOM_ESW_OFFLOADS,
        MLX5_DEVCOM_MPV,
+       MLX5_DEVCOM_HCA_PORTS,
        MLX5_DEVCOM_NUM_COMPONENTS,
 };
 
@@ -52,4 +53,7 @@ void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom,
             data;                                                                \
             data = mlx5_devcom_get_next_peer_data_rcu(devcom, &pos))
 
+void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom);
+void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom);
+
 #endif /* __LIB_MLX5_DEVCOM_H__ */

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 307ffe6300f8e96c0ed15367e21b78fc9287d368..a17152c1cbb247efa46b875698cf2eac3738e82a 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -73,6 +73,7 @@
 #include "sf/sf.h"
 #include "mlx5_irq.h"
 #include "hwmon.h"
+#include "lag/lag.h"
 
 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
 MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver");
@@ -952,6 +953,27 @@ static void mlx5_pci_close(struct mlx5_core_dev *dev)
        mlx5_pci_disable_device(dev);
 }
 
+static void mlx5_register_hca_devcom_comp(struct mlx5_core_dev *dev)
+{
+       /* This component is use to sync adding core_dev to lag_dev and to sync
+        * changes of mlx5_adev_devices between LAG layer and other layers.
+        */
+       if (!mlx5_lag_is_supported(dev))
+               return;
+
+       dev->priv.hca_devcom_comp =
+               mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_HCA_PORTS,
+                                              mlx5_query_nic_system_image_guid(dev),
+                                              NULL, dev);
+       if (IS_ERR_OR_NULL(dev->priv.hca_devcom_comp))
+               mlx5_core_err(dev, "Failed to register devcom HCA component\n");
+}
+
+static void mlx5_unregister_hca_devcom_comp(struct mlx5_core_dev *dev)
+{
+       mlx5_devcom_unregister_component(dev->priv.hca_devcom_comp);
+}
+
 static int mlx5_init_once(struct mlx5_core_dev *dev)
 {
        int err;
@@ -960,6 +982,7 @@ static int mlx5_init_once(struct mlx5_core_dev *dev)
        if (IS_ERR(dev->priv.devc))
                mlx5_core_warn(dev, "failed to register devcom device %ld\n",
                               PTR_ERR(dev->priv.devc));
+       mlx5_register_hca_devcom_comp(dev);
 
        err = mlx5_query_board_id(dev);
        if (err) {
@@ -1094,6 +1117,7 @@ static int mlx5_init_once(struct mlx5_core_dev *dev)
 err_irq_cleanup:
        mlx5_irq_table_cleanup(dev);
 err_devcom:
+       mlx5_unregister_hca_devcom_comp(dev);
        mlx5_devcom_unregister_device(dev->priv.devc);
 
        return err;
@@ -1123,6 +1147,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
        mlx5_events_cleanup(dev);
        mlx5_eq_table_cleanup(dev);
        mlx5_irq_table_cleanup(dev);
+       mlx5_unregister_hca_devcom_comp(dev);
        mlx5_devcom_unregister_device(dev->priv.devc);
 }
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index d348a7f9511f1a1ec019d90ba0f9308872389610..f191a90f337b8a844716497236b8f6be629e8c2c 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -266,7 +266,6 @@ int mlx5_register_device(struct mlx5_core_dev *dev);
 void mlx5_unregister_device(struct mlx5_core_dev *dev);
 void mlx5_dev_set_lightweight(struct mlx5_core_dev *dev);
 bool mlx5_dev_is_lightweight(struct mlx5_core_dev *dev);
-struct mlx5_core_dev *mlx5_get_next_phys_dev_lag(struct mlx5_core_dev *dev);
 void mlx5_dev_list_lock(void);
 void mlx5_dev_list_unlock(void);
 int mlx5_dev_list_trylock(void);

diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 7968c5ee85c41345d287a1070bd4a3f833722f7b..f60cdc9bd40f198eb8ddfc050a62704dcd651268 100644 (file)
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -624,6 +624,7 @@ struct mlx5_priv {
        struct mlx5_lag         *lag;
        u32                     flags;
        struct mlx5_devcom_dev  *devc;
+       struct mlx5_devcom_comp_dev *hca_devcom_comp;
        struct mlx5_fw_reset    *fw_reset;
        struct mlx5_core_roce   roce;
        struct mlx5_fc_stats            fc_stats;