mlx5_cmd_out_err(dev->mdev, MLX5_CMD_OP_CREATE_MKEY, 0, out);
}
-static int push_mkey_locked(struct mlx5_cache_ent *ent, bool limit_pendings,
- void *to_store)
+static int push_mkey_locked(struct mlx5_cache_ent *ent, u32 mkey)
{
- XA_STATE(xas, &ent->mkeys, 0);
- void *curr;
+ unsigned long tmp = ent->mkeys_queue.ci % NUM_MKEYS_PER_PAGE;
+ struct mlx5_mkeys_page *page;
- if (limit_pendings &&
- (ent->reserved - ent->stored) > MAX_PENDING_REG_MR)
- return -EAGAIN;
-
- while (1) {
- /*
- * This is cmpxchg (NULL, XA_ZERO_ENTRY) however this version
- * doesn't transparently unlock. Instead we set the xas index to
- * the current value of reserved every iteration.
- */
- xas_set(&xas, ent->reserved);
- curr = xas_load(&xas);
- if (!curr) {
- if (to_store && ent->stored == ent->reserved)
- xas_store(&xas, to_store);
- else
- xas_store(&xas, XA_ZERO_ENTRY);
- if (xas_valid(&xas)) {
- ent->reserved++;
- if (to_store) {
- if (ent->stored != ent->reserved)
- __xa_store(&ent->mkeys,
- ent->stored,
- to_store,
- GFP_KERNEL);
- ent->stored++;
- queue_adjust_cache_locked(ent);
- WRITE_ONCE(ent->dev->cache.last_add,
- jiffies);
- }
- }
- }
- xa_unlock_irq(&ent->mkeys);
-
- /*
- * Notice xas_nomem() must always be called as it cleans
- * up any cached allocation.
- */
- if (!xas_nomem(&xas, GFP_KERNEL))
- break;
- xa_lock_irq(&ent->mkeys);
+ lockdep_assert_held(&ent->mkeys_queue.lock);
+ if (ent->mkeys_queue.ci >=
+ ent->mkeys_queue.num_pages * NUM_MKEYS_PER_PAGE) {
+ page = kzalloc(sizeof(*page), GFP_ATOMIC);
+ if (!page)
+ return -ENOMEM;
+ ent->mkeys_queue.num_pages++;
+ list_add_tail(&page->list, &ent->mkeys_queue.pages_list);
+ } else {
+ page = list_last_entry(&ent->mkeys_queue.pages_list,
+ struct mlx5_mkeys_page, list);
}
- xa_lock_irq(&ent->mkeys);
- if (xas_error(&xas))
- return xas_error(&xas);
- if (WARN_ON(curr))
- return -EINVAL;
- return 0;
-}
-
-static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings,
- void *to_store)
-{
- int ret;
-
- xa_lock_irq(&ent->mkeys);
- ret = push_mkey_locked(ent, limit_pendings, to_store);
- xa_unlock_irq(&ent->mkeys);
- return ret;
-}
-
-static void undo_push_reserve_mkey(struct mlx5_cache_ent *ent)
-{
- void *old;
-
- ent->reserved--;
- old = __xa_erase(&ent->mkeys, ent->reserved);
- WARN_ON(old);
-}
-
-static void push_to_reserved(struct mlx5_cache_ent *ent, u32 mkey)
-{
- void *old;
- old = __xa_store(&ent->mkeys, ent->stored, xa_mk_value(mkey), 0);
- WARN_ON(old);
- ent->stored++;
+ page->mkeys[tmp] = mkey;
+ ent->mkeys_queue.ci++;
+ return 0;
}
-static u32 pop_stored_mkey(struct mlx5_cache_ent *ent)
+static int pop_mkey_locked(struct mlx5_cache_ent *ent)
{
- void *old, *xa_mkey;
-
- ent->stored--;
- ent->reserved--;
+ unsigned long tmp = (ent->mkeys_queue.ci - 1) % NUM_MKEYS_PER_PAGE;
+ struct mlx5_mkeys_page *last_page;
+ u32 mkey;
- if (ent->stored == ent->reserved) {
- xa_mkey = __xa_erase(&ent->mkeys, ent->stored);
- WARN_ON(!xa_mkey);
- return (u32)xa_to_value(xa_mkey);
+ lockdep_assert_held(&ent->mkeys_queue.lock);
+ last_page = list_last_entry(&ent->mkeys_queue.pages_list,
+ struct mlx5_mkeys_page, list);
+ mkey = last_page->mkeys[tmp];
+ last_page->mkeys[tmp] = 0;
+ ent->mkeys_queue.ci--;
+ if (ent->mkeys_queue.num_pages > 1 && !tmp) {
+ list_del(&last_page->list);
+ ent->mkeys_queue.num_pages--;
+ kfree(last_page);
}
-
- xa_mkey = __xa_store(&ent->mkeys, ent->stored, XA_ZERO_ENTRY,
- GFP_KERNEL);
- WARN_ON(!xa_mkey || xa_is_err(xa_mkey));
- old = __xa_erase(&ent->mkeys, ent->reserved);
- WARN_ON(old);
- return (u32)xa_to_value(xa_mkey);
+ return mkey;
}
static void create_mkey_callback(int status, struct mlx5_async_work *context)
if (status) {
create_mkey_warn(dev, status, mkey_out->out);
kfree(mkey_out);
- xa_lock_irqsave(&ent->mkeys, flags);
- undo_push_reserve_mkey(ent);
+ spin_lock_irqsave(&ent->mkeys_queue.lock, flags);
+ ent->pending--;
WRITE_ONCE(dev->fill_delay, 1);
- xa_unlock_irqrestore(&ent->mkeys, flags);
+ spin_unlock_irqrestore(&ent->mkeys_queue.lock, flags);
mod_timer(&dev->delay_timer, jiffies + HZ);
return;
}
MLX5_GET(create_mkey_out, mkey_out->out, mkey_index));
WRITE_ONCE(dev->cache.last_add, jiffies);
- xa_lock_irqsave(&ent->mkeys, flags);
- push_to_reserved(ent, mkey_out->mkey);
+ spin_lock_irqsave(&ent->mkeys_queue.lock, flags);
+ push_mkey_locked(ent, mkey_out->mkey);
/* If we are doing fill_to_high_water then keep going. */
queue_adjust_cache_locked(ent);
- xa_unlock_irqrestore(&ent->mkeys, flags);
+ ent->pending--;
+ spin_unlock_irqrestore(&ent->mkeys_queue.lock, flags);
kfree(mkey_out);
}
set_cache_mkc(ent, mkc);
async_create->ent = ent;
- err = push_mkey(ent, true, NULL);
- if (err)
+ spin_lock_irq(&ent->mkeys_queue.lock);
+ if (ent->pending >= MAX_PENDING_REG_MR) {
+ err = -EAGAIN;
goto free_async_create;
+ }
+ ent->pending++;
+ spin_unlock_irq(&ent->mkeys_queue.lock);
err = mlx5_ib_create_mkey_cb(async_create);
if (err) {
mlx5_ib_warn(ent->dev, "create mkey failed %d\n", err);
- goto err_undo_reserve;
+ goto err_create_mkey;
}
}
return 0;
-err_undo_reserve:
- xa_lock_irq(&ent->mkeys);
- undo_push_reserve_mkey(ent);
- xa_unlock_irq(&ent->mkeys);
+err_create_mkey:
+ spin_lock_irq(&ent->mkeys_queue.lock);
+ ent->pending--;
free_async_create:
+ spin_unlock_irq(&ent->mkeys_queue.lock);
kfree(async_create);
return err;
}
{
u32 mkey;
- lockdep_assert_held(&ent->mkeys.xa_lock);
- if (!ent->stored)
+ lockdep_assert_held(&ent->mkeys_queue.lock);
+ if (!ent->mkeys_queue.ci)
return;
- mkey = pop_stored_mkey(ent);
- xa_unlock_irq(&ent->mkeys);
+ mkey = pop_mkey_locked(ent);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
mlx5_core_destroy_mkey(ent->dev->mdev, mkey);
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
}
static int resize_available_mrs(struct mlx5_cache_ent *ent, unsigned int target,
bool limit_fill)
- __acquires(&ent->mkeys) __releases(&ent->mkeys)
+ __acquires(&ent->mkeys_queue.lock) __releases(&ent->mkeys_queue.lock)
{
int err;
- lockdep_assert_held(&ent->mkeys.xa_lock);
+ lockdep_assert_held(&ent->mkeys_queue.lock);
while (true) {
if (limit_fill)
target = ent->limit * 2;
- if (target == ent->reserved)
+ if (target == ent->pending + ent->mkeys_queue.ci)
return 0;
- if (target > ent->reserved) {
- u32 todo = target - ent->reserved;
+ if (target > ent->pending + ent->mkeys_queue.ci) {
+ u32 todo = target - (ent->pending + ent->mkeys_queue.ci);
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
err = add_keys(ent, todo);
if (err == -EAGAIN)
usleep_range(3000, 5000);
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
if (err) {
if (err != -EAGAIN)
return err;
* cannot free MRs that are in use. Compute the target value for stored
* mkeys.
*/
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
if (target < ent->in_use) {
err = -EINVAL;
goto err_unlock;
err = resize_available_mrs(ent, target, false);
if (err)
goto err_unlock;
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
return count;
err_unlock:
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
return err;
}
char lbuf[20];
int err;
- err = snprintf(lbuf, sizeof(lbuf), "%ld\n", ent->stored + ent->in_use);
+ err = snprintf(lbuf, sizeof(lbuf), "%ld\n",
+ ent->mkeys_queue.ci + ent->in_use);
if (err < 0)
return err;
* Upon set we immediately fill the cache to high water mark implied by
* the limit.
*/
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
ent->limit = var;
err = resize_available_mrs(ent, 0, true);
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
if (err)
return err;
return count;
mutex_lock(&cache->rb_lock);
for (node = rb_first(&cache->rb_root); node; node = rb_next(node)) {
ent = rb_entry(node, struct mlx5_cache_ent, node);
- xa_lock_irq(&ent->mkeys);
- ret = ent->stored < ent->limit;
- xa_unlock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
+ ret = ent->mkeys_queue.ci < ent->limit;
+ spin_unlock_irq(&ent->mkeys_queue.lock);
if (ret) {
mutex_unlock(&cache->rb_lock);
return true;
*/
static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)
{
- lockdep_assert_held(&ent->mkeys.xa_lock);
+ lockdep_assert_held(&ent->mkeys_queue.lock);
if (ent->disabled || READ_ONCE(ent->dev->fill_delay) || ent->is_tmp)
return;
- if (ent->stored < ent->limit) {
+ if (ent->mkeys_queue.ci < ent->limit) {
ent->fill_to_high_water = true;
mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);
} else if (ent->fill_to_high_water &&
- ent->reserved < 2 * ent->limit) {
+ ent->mkeys_queue.ci + ent->pending < 2 * ent->limit) {
/*
* Once we start populating due to hitting a low water mark
* continue until we pass the high water mark.
*/
mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);
- } else if (ent->stored == 2 * ent->limit) {
+ } else if (ent->mkeys_queue.ci == 2 * ent->limit) {
ent->fill_to_high_water = false;
- } else if (ent->stored > 2 * ent->limit) {
+ } else if (ent->mkeys_queue.ci > 2 * ent->limit) {
/* Queue deletion of excess entries */
ent->fill_to_high_water = false;
- if (ent->stored != ent->reserved)
+ if (ent->pending)
queue_delayed_work(ent->dev->cache.wq, &ent->dwork,
msecs_to_jiffies(1000));
else
struct mlx5_mkey_cache *cache = &dev->cache;
int err;
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
if (ent->disabled)
goto out;
- if (ent->fill_to_high_water && ent->reserved < 2 * ent->limit &&
+ if (ent->fill_to_high_water &&
+ ent->mkeys_queue.ci + ent->pending < 2 * ent->limit &&
!READ_ONCE(dev->fill_delay)) {
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
err = add_keys(ent, 1);
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
if (ent->disabled)
goto out;
if (err) {
msecs_to_jiffies(1000));
}
}
- } else if (ent->stored > 2 * ent->limit) {
+ } else if (ent->mkeys_queue.ci > 2 * ent->limit) {
bool need_delay;
/*
* the garbage collection work to try to run in next cycle, in
* order to free CPU resources to other tasks.
*/
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
need_delay = need_resched() || someone_adding(cache) ||
!time_after(jiffies,
READ_ONCE(cache->last_add) + 300 * HZ);
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
if (ent->disabled)
goto out;
if (need_delay) {
queue_adjust_cache_locked(ent);
}
out:
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
}
static void delayed_cache_work_func(struct work_struct *work)
if (!mr)
return ERR_PTR(-ENOMEM);
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
ent->in_use++;
- if (!ent->stored) {
+ if (!ent->mkeys_queue.ci) {
queue_adjust_cache_locked(ent);
ent->miss++;
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
err = create_cache_mkey(ent, &mr->mmkey.key);
if (err) {
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
ent->in_use--;
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
kfree(mr);
return ERR_PTR(err);
}
} else {
- mr->mmkey.key = pop_stored_mkey(ent);
+ mr->mmkey.key = pop_mkey_locked(ent);
queue_adjust_cache_locked(ent);
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
}
mr->mmkey.cache_ent = ent;
mr->mmkey.type = MLX5_MKEY_MR;
u32 mkey;
cancel_delayed_work(&ent->dwork);
- xa_lock_irq(&ent->mkeys);
- while (ent->stored) {
- mkey = pop_stored_mkey(ent);
- xa_unlock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
+ while (ent->mkeys_queue.ci) {
+ mkey = pop_mkey_locked(ent);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
mlx5_core_destroy_mkey(dev->mdev, mkey);
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
}
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
}
static void mlx5_mkey_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
dir = debugfs_create_dir(ent->name, dev->cache.fs_root);
debugfs_create_file("size", 0600, dir, ent, &size_fops);
debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
- debugfs_create_ulong("cur", 0400, dir, &ent->stored);
+ debugfs_create_ulong("cur", 0400, dir, &ent->mkeys_queue.ci);
debugfs_create_u32("miss", 0600, dir, &ent->miss);
}
WRITE_ONCE(dev->fill_delay, 0);
}
+static int mlx5r_mkeys_init(struct mlx5_cache_ent *ent)
+{
+ struct mlx5_mkeys_page *page;
+
+ page = kzalloc(sizeof(*page), GFP_KERNEL);
+ if (!page)
+ return -ENOMEM;
+ INIT_LIST_HEAD(&ent->mkeys_queue.pages_list);
+ spin_lock_init(&ent->mkeys_queue.lock);
+ list_add_tail(&page->list, &ent->mkeys_queue.pages_list);
+ ent->mkeys_queue.num_pages++;
+ return 0;
+}
+
+static void mlx5r_mkeys_uninit(struct mlx5_cache_ent *ent)
+{
+ struct mlx5_mkeys_page *page;
+
+ WARN_ON(ent->mkeys_queue.ci || ent->mkeys_queue.num_pages > 1);
+ page = list_last_entry(&ent->mkeys_queue.pages_list,
+ struct mlx5_mkeys_page, list);
+ list_del(&page->list);
+ kfree(page);
+}
+
struct mlx5_cache_ent *
mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev,
struct mlx5r_cache_rb_key rb_key,
if (!ent)
return ERR_PTR(-ENOMEM);
- xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ);
+ ret = mlx5r_mkeys_init(ent);
+ if (ret)
+ goto mkeys_err;
ent->rb_key = rb_key;
ent->dev = dev;
ent->is_tmp = !persistent_entry;
INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
ret = mlx5_cache_ent_insert(&dev->cache, ent);
- if (ret) {
- kfree(ent);
- return ERR_PTR(ret);
- }
+ if (ret)
+ goto ent_insert_err;
if (persistent_entry) {
if (rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM)
}
return ent;
+ent_insert_err:
+ mlx5r_mkeys_uninit(ent);
+mkeys_err:
+ kfree(ent);
+ return ERR_PTR(ret);
}
static void remove_ent_work_func(struct work_struct *work)
cur = rb_prev(cur);
mutex_unlock(&cache->rb_lock);
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
if (!ent->is_tmp) {
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
mutex_lock(&cache->rb_lock);
continue;
}
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
clean_keys(ent->dev, ent);
mutex_lock(&cache->rb_lock);
mutex_unlock(&cache->rb_lock);
for (node = rb_first(root); node; node = rb_next(node)) {
ent = rb_entry(node, struct mlx5_cache_ent, node);
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
queue_adjust_cache_locked(ent);
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
}
return 0;
mutex_lock(&dev->cache.rb_lock);
for (node = rb_first(root); node; node = rb_next(node)) {
ent = rb_entry(node, struct mlx5_cache_ent, node);
- xa_lock_irq(&ent->mkeys);
+ spin_lock_irq(&ent->mkeys_queue.lock);
ent->disabled = true;
- xa_unlock_irq(&ent->mkeys);
+ spin_unlock_irq(&ent->mkeys_queue.lock);
cancel_delayed_work_sync(&ent->dwork);
}
node = rb_next(node);
clean_keys(dev, ent);
rb_erase(&ent->node, root);
+ mlx5r_mkeys_uninit(ent);
kfree(ent);
}
mutex_unlock(&dev->cache.rb_lock);
int ret;
if (mr->mmkey.cache_ent) {
- xa_lock_irq(&mr->mmkey.cache_ent->mkeys);
+ spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);
mr->mmkey.cache_ent->in_use--;
goto end;
}
return -EOPNOTSUPP;
}
mr->mmkey.cache_ent = ent;
- xa_lock_irq(&mr->mmkey.cache_ent->mkeys);
+ spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);
mutex_unlock(&cache->rb_lock);
goto end;
}
return PTR_ERR(ent);
mr->mmkey.cache_ent = ent;
- xa_lock_irq(&mr->mmkey.cache_ent->mkeys);
+ spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);
end:
- ret = push_mkey_locked(mr->mmkey.cache_ent, false,
- xa_mk_value(mr->mmkey.key));
- xa_unlock_irq(&mr->mmkey.cache_ent->mkeys);
+ ret = push_mkey_locked(mr->mmkey.cache_ent, mr->mmkey.key);
+ spin_unlock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);
return ret;
}
projects / linux-stable / commitdiff