dm io: Support IO priority

[ Upstream commit 6e5f0f6383b4896c7e9b943d84b136149d0f45e9 ]

Some IO will dispatch from kworker with different io_context settings
than the submitting task, we may need to specify a priority to avoid
losing priority.

Add IO priority parameter to dm_io() and update all callers.

Co-developed-by: Yibin Ding <yibin.ding@unisoc.com>
Signed-off-by: Yibin Ding <yibin.ding@unisoc.com>
Signed-off-by: Hongyu Jin <hongyu.jin@unisoc.com>
Reviewed-by: Eric Biggers <ebiggers@google.com>
Reviewed-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Mike Snitzer <snitzer@kernel.org>
Stable-dep-of: b4d78cfeb304 ("dm-integrity: align the outgoing bio in integrity_recheck")
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index 13c65b7e1ed63520c3227804ecaae2e1de6dbf90..f5541b8f6320c9a5db6bb5c22a3d0d59d1f3ec3c 100644 (file)
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -1315,7 +1315,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector,
                io_req.mem.ptr.vma = (char *)b->data + offset;
        }
 
-       r = dm_io(&io_req, 1, &region, NULL);
+       r = dm_io(&io_req, 1, &region, NULL, IOPRIO_DEFAULT);
        if (unlikely(r))
                b->end_io(b, errno_to_blk_status(r));
 }
@@ -2167,7 +2167,7 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c)
        if (WARN_ON_ONCE(dm_bufio_in_request()))
                return -EINVAL;
 
-       return dm_io(&io_req, 1, &io_reg, NULL);
+       return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_issue_flush);
 
@@ -2191,7 +2191,7 @@ int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t c
        if (WARN_ON_ONCE(dm_bufio_in_request()))
                return -EINVAL; /* discards are optional */
 
-       return dm_io(&io_req, 1, &io_reg, NULL);
+       return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_issue_discard);
 

diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
index f194ce6bfacb5cb964d76f04f8c975b2b60c2604..92bed6ce17931116756f66abe9d58b27cdff020f 100644 (file)
--- a/drivers/md/dm-integrity.c
+++ b/drivers/md/dm-integrity.c
@@ -555,7 +555,7 @@ static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf)
                }
        }
 
-       r = dm_io(&io_req, 1, &io_loc, NULL);
+       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
        if (unlikely(r))
                return r;
 
@@ -1073,7 +1073,7 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf,
        io_loc.sector = ic->start + SB_SECTORS + sector;
        io_loc.count = n_sectors;
 
-       r = dm_io(&io_req, 1, &io_loc, NULL);
+       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
        if (unlikely(r)) {
                dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ?
                                      "reading journal" : "writing journal", r);
@@ -1190,7 +1190,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, u
        io_loc.sector = target;
        io_loc.count = n_sectors;
 
-       r = dm_io(&io_req, 1, &io_loc, NULL);
+       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
        if (unlikely(r)) {
                WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
                fn(-1UL, data);
@@ -1519,7 +1519,7 @@ static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_dat
                fr.io_reg.count = 0,
                fr.ic = ic;
                init_completion(&fr.comp);
-               r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL);
+               r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT);
                BUG_ON(r);
        }
 
@@ -1727,7 +1727,7 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks
                        io_loc.sector = sector;
                        io_loc.count = ic->sectors_per_block;
 
-                       r = dm_io(&io_req, 1, &io_loc, NULL);
+                       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
                        if (unlikely(r)) {
                                dio->bi_status = errno_to_blk_status(r);
                                goto free_ret;
@@ -2806,7 +2806,7 @@ static void integrity_recalc(struct work_struct *w)
        io_loc.sector = get_data_sector(ic, area, offset);
        io_loc.count = n_sectors;
 
-       r = dm_io(&io_req, 1, &io_loc, NULL);
+       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
        if (unlikely(r)) {
                dm_integrity_io_error(ic, "reading data", r);
                goto err;

diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
index f053ce2458147c5e7efb31a2aff9b7f66a169868..7409490259d1d4e0407a1b38e4d3f3cdfc33b36c 100644 (file)
--- a/drivers/md/dm-io.c
+++ b/drivers/md/dm-io.c
@@ -305,7 +305,7 @@ static void km_dp_init(struct dpages *dp, void *data)
  */
 static void do_region(const blk_opf_t opf, unsigned int region,
                      struct dm_io_region *where, struct dpages *dp,
-                     struct io *io)
+                     struct io *io, unsigned short ioprio)
 {
        struct bio *bio;
        struct page *page;
@@ -354,6 +354,7 @@ static void do_region(const blk_opf_t opf, unsigned int region,
                                       &io->client->bios);
                bio->bi_iter.bi_sector = where->sector + (where->count - remaining);
                bio->bi_end_io = endio;
+               bio->bi_ioprio = ioprio;
                store_io_and_region_in_bio(bio, io, region);
 
                if (op == REQ_OP_DISCARD || op == REQ_OP_WRITE_ZEROES) {
@@ -383,7 +384,7 @@ static void do_region(const blk_opf_t opf, unsigned int region,
 
 static void dispatch_io(blk_opf_t opf, unsigned int num_regions,
                        struct dm_io_region *where, struct dpages *dp,
-                       struct io *io, int sync)
+                       struct io *io, int sync, unsigned short ioprio)
 {
        int i;
        struct dpages old_pages = *dp;
@@ -400,7 +401,7 @@ static void dispatch_io(blk_opf_t opf, unsigned int num_regions,
        for (i = 0; i < num_regions; i++) {
                *dp = old_pages;
                if (where[i].count || (opf & REQ_PREFLUSH))
-                       do_region(opf, i, where + i, dp, io);
+                       do_region(opf, i, where + i, dp, io, ioprio);
        }
 
        /*
@@ -425,7 +426,7 @@ static void sync_io_complete(unsigned long error, void *context)
 
 static int sync_io(struct dm_io_client *client, unsigned int num_regions,
                   struct dm_io_region *where, blk_opf_t opf, struct dpages *dp,
-                  unsigned long *error_bits)
+                  unsigned long *error_bits, unsigned short ioprio)
 {
        struct io *io;
        struct sync_io sio;
@@ -447,7 +448,7 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions,
        io->vma_invalidate_address = dp->vma_invalidate_address;
        io->vma_invalidate_size = dp->vma_invalidate_size;
 
-       dispatch_io(opf, num_regions, where, dp, io, 1);
+       dispatch_io(opf, num_regions, where, dp, io, 1, ioprio);
 
        wait_for_completion_io(&sio.wait);
 
@@ -459,7 +460,8 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions,
 
 static int async_io(struct dm_io_client *client, unsigned int num_regions,
                    struct dm_io_region *where, blk_opf_t opf,
-                   struct dpages *dp, io_notify_fn fn, void *context)
+                   struct dpages *dp, io_notify_fn fn, void *context,
+                   unsigned short ioprio)
 {
        struct io *io;
 
@@ -479,7 +481,7 @@ static int async_io(struct dm_io_client *client, unsigned int num_regions,
        io->vma_invalidate_address = dp->vma_invalidate_address;
        io->vma_invalidate_size = dp->vma_invalidate_size;
 
-       dispatch_io(opf, num_regions, where, dp, io, 0);
+       dispatch_io(opf, num_regions, where, dp, io, 0, ioprio);
        return 0;
 }
 
@@ -521,7 +523,8 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp,
 }
 
 int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
-         struct dm_io_region *where, unsigned long *sync_error_bits)
+         struct dm_io_region *where, unsigned long *sync_error_bits,
+         unsigned short ioprio)
 {
        int r;
        struct dpages dp;
@@ -532,11 +535,11 @@ int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
 
        if (!io_req->notify.fn)
                return sync_io(io_req->client, num_regions, where,
-                              io_req->bi_opf, &dp, sync_error_bits);
+                              io_req->bi_opf, &dp, sync_error_bits, ioprio);
 
        return async_io(io_req->client, num_regions, where,
                        io_req->bi_opf, &dp, io_req->notify.fn,
-                       io_req->notify.context);
+                       io_req->notify.context, ioprio);
 }
 EXPORT_SYMBOL(dm_io);
 

diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c
index 36bcfdccae0461d6c14e794785c8bf31755719b5..6ea75436a433aa6a48560ce3a6f517f4e9ac8221 100644 (file)
--- a/drivers/md/dm-kcopyd.c
+++ b/drivers/md/dm-kcopyd.c
@@ -578,9 +578,9 @@ static int run_io_job(struct kcopyd_job *job)
        io_job_start(job->kc->throttle);
 
        if (job->op == REQ_OP_READ)
-               r = dm_io(&io_req, 1, &job->source, NULL);
+               r = dm_io(&io_req, 1, &job->source, NULL, IOPRIO_DEFAULT);
        else
-               r = dm_io(&io_req, job->num_dests, job->dests, NULL);
+               r = dm_io(&io_req, job->num_dests, job->dests, NULL, IOPRIO_DEFAULT);
 
        return r;
 }

diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c
index f9f84236dfcd750e0464d6a8e17c0b9d0f7109d4..f7f9c2100937baaf3c0aa876b29e9e7856dbe4d1 100644 (file)
--- a/drivers/md/dm-log.c
+++ b/drivers/md/dm-log.c
@@ -300,7 +300,7 @@ static int rw_header(struct log_c *lc, enum req_op op)
 {
        lc->io_req.bi_opf = op;
 
-       return dm_io(&lc->io_req, 1, &lc->header_location, NULL);
+       return dm_io(&lc->io_req, 1, &lc->header_location, NULL, IOPRIO_DEFAULT);
 }
 
 static int flush_header(struct log_c *lc)
@@ -313,7 +313,7 @@ static int flush_header(struct log_c *lc)
 
        lc->io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 
-       return dm_io(&lc->io_req, 1, &null_location, NULL);
+       return dm_io(&lc->io_req, 1, &null_location, NULL, IOPRIO_DEFAULT);
 }
 
 static int read_header(struct log_c *log)

diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index ddcb2bc4a6179ac6552f9f627a1de3625edfdba7..9511dae5b556a9994501392cac3766aa129f15a7 100644 (file)
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -278,7 +278,7 @@ static int mirror_flush(struct dm_target *ti)
        }
 
        error_bits = -1;
-       dm_io(&io_req, ms->nr_mirrors, io, &error_bits);
+       dm_io(&io_req, ms->nr_mirrors, io, &error_bits, IOPRIO_DEFAULT);
        if (unlikely(error_bits != 0)) {
                for (i = 0; i < ms->nr_mirrors; i++)
                        if (test_bit(i, &error_bits))
@@ -554,7 +554,7 @@ static void read_async_bio(struct mirror *m, struct bio *bio)
 
        map_region(&io, m, bio);
        bio_set_m(bio, m);
-       BUG_ON(dm_io(&io_req, 1, &io, NULL));
+       BUG_ON(dm_io(&io_req, 1, &io, NULL, IOPRIO_DEFAULT));
 }
 
 static inline int region_in_sync(struct mirror_set *ms, region_t region,
@@ -681,7 +681,7 @@ static void do_write(struct mirror_set *ms, struct bio *bio)
         */
        bio_set_m(bio, get_default_mirror(ms));
 
-       BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL));
+       BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, IOPRIO_DEFAULT));
 }
 
 static void do_writes(struct mirror_set *ms, struct bio_list *writes)

diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c
index 15649921f2a9b1eb8eb0c1e0fb2897d074dd238c..568d10842b1f46f31ff009e129c2069cd2812f26 100644 (file)
--- a/drivers/md/dm-snap-persistent.c
+++ b/drivers/md/dm-snap-persistent.c
@@ -223,7 +223,7 @@ static void do_metadata(struct work_struct *work)
 {
        struct mdata_req *req = container_of(work, struct mdata_req, work);
 
-       req->result = dm_io(req->io_req, 1, req->where, NULL);
+       req->result = dm_io(req->io_req, 1, req->where, NULL, IOPRIO_DEFAULT);
 }
 
 /*
@@ -247,7 +247,7 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, blk_opf_t opf,
        struct mdata_req req;
 
        if (!metadata)
-               return dm_io(&io_req, 1, &where, NULL);
+               return dm_io(&io_req, 1, &where, NULL, IOPRIO_DEFAULT);
 
        req.where = &where;
        req.io_req = &io_req;

diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index 1b591bfa90d5d6463016e22183dbb5f071e94a75..abc008bae9020cca7b926a2e93add189ea998640 100644 (file)
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -511,7 +511,7 @@ static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io,
        io_loc.bdev = v->data_dev->bdev;
        io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT);
        io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT);
-       r = dm_io(&io_req, 1, &io_loc, NULL);
+       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
        if (unlikely(r))
                goto free_ret;
 

diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c
index b463c28c39ad34ca23b3d2433811384901171d80..01ab141bc51e3c69de2d3e6b2a4195f73fd91e74 100644 (file)
--- a/drivers/md/dm-writecache.c
+++ b/drivers/md/dm-writecache.c
@@ -531,7 +531,7 @@ static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
                req.notify.context = &endio;
 
                /* writing via async dm-io (implied by notify.fn above) won't return an error */
-               (void) dm_io(&req, 1, &region, NULL);
+               (void) dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
                i = j;
        }
 
@@ -568,7 +568,7 @@ static void ssd_commit_superblock(struct dm_writecache *wc)
        req.notify.fn = NULL;
        req.notify.context = NULL;
 
-       r = dm_io(&req, 1, &region, NULL);
+       r = dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
        if (unlikely(r))
                writecache_error(wc, r, "error writing superblock");
 }
@@ -596,7 +596,7 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
        req.client = wc->dm_io;
        req.notify.fn = NULL;
 
-       r = dm_io(&req, 1, &region, NULL);
+       r = dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
        if (unlikely(r))
                writecache_error(wc, r, "error flushing metadata: %d", r);
 }
@@ -990,7 +990,7 @@ static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors
        req.client = wc->dm_io;
        req.notify.fn = NULL;
 
-       return dm_io(&req, 1, &region, NULL);
+       return dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
 }
 
 static void writecache_resume(struct dm_target *ti)

diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h
index 7595142f3fc57da9e7d27c8f7ad72f79b86f8692..7b2968612b7e69d01a7f82d600c0e13546a98c1e 100644 (file)
--- a/include/linux/dm-io.h
+++ b/include/linux/dm-io.h
@@ -80,7 +80,8 @@ void dm_io_client_destroy(struct dm_io_client *client);
  * error occurred doing io to the corresponding region.
  */
 int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
-         struct dm_io_region *region, unsigned int long *sync_error_bits);
+         struct dm_io_region *region, unsigned int long *sync_error_bits,
+         unsigned short ioprio);
 
 #endif /* __KERNEL__ */
 #endif /* _LINUX_DM_IO_H */