io_uring/rsrc: improve code generation for fixed file assignment

For the normal read/write path, we have already locked the ring
submission side when assigning the file. This causes branch
mispredictions when we then check and try and lock again in
io_req_set_rsrc_node(). As this is a very hot path, this matters.

Add a basic helper that already assumes we already have it locked,
and use that in io_file_get_fixed().

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 4afb911fc042bda0a6c7a7e3827da9e902122aa3..50c9f04bc193c5ca6953372d945b9ba67c07539f 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2000,9 +2000,10 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
                goto out;
        fd = array_index_nospec(fd, ctx->nr_user_files);
        slot = io_fixed_file_slot(&ctx->file_table, fd);
-       file = io_slot_file(slot);
+       if (!req->rsrc_node)
+               __io_req_set_rsrc_node(req, ctx);
        req->flags |= io_slot_flags(slot);
-       io_req_set_rsrc_node(req, ctx, 0);
+       file = io_slot_file(slot);
 out:
        io_ring_submit_unlock(ctx, issue_flags);
        return file;

diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 7238b9cfe33b60b7520905d7f632951212c9677c..c6f199bbee2843dfea2d88729b707d46a168d3d9 100644 (file)
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -102,17 +102,21 @@ static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx,
        node->refs++;
 }
 
+static inline void __io_req_set_rsrc_node(struct io_kiocb *req,
+                                         struct io_ring_ctx *ctx)
+{
+       lockdep_assert_held(&ctx->uring_lock);
+       req->rsrc_node = ctx->rsrc_node;
+       io_charge_rsrc_node(ctx, ctx->rsrc_node);
+}
+
 static inline void io_req_set_rsrc_node(struct io_kiocb *req,
                                        struct io_ring_ctx *ctx,
                                        unsigned int issue_flags)
 {
        if (!req->rsrc_node) {
                io_ring_submit_lock(ctx, issue_flags);
-
-               lockdep_assert_held(&ctx->uring_lock);
-
-               req->rsrc_node = ctx->rsrc_node;
-               io_charge_rsrc_node(ctx, ctx->rsrc_node);
+               __io_req_set_rsrc_node(req, ctx);
                io_ring_submit_unlock(ctx, issue_flags);
        }
 }