io_uring: Fix release of pinned pages when __io_uaddr_map fails

[ Upstream commit 67d1189d1095d471ed7fa426c7e384a7140a5dd7 ]

Looking at the error path of __io_uaddr_map, if we fail after pinning
the pages for any reasons, ret will be set to -EINVAL and the error
handler won't properly release the pinned pages.

I didn't manage to trigger it without forcing a failure, but it can
happen in real life when memory is heavily fragmented.

Signed-off-by: Gabriel Krisman Bertazi <krisman@suse.de>
Fixes: 223ef4743164 ("io_uring: don't allow IORING_SETUP_NO_MMAP rings on highmem pages")
Link: https://lore.kernel.org/r/20240313213912.1920-1-krisman@suse.de
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 030cc930d1c6a6b014d7ae5978063ab6a2a293fc..9f938874c5e1356b056864c2c14e243d26f0b03f 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2702,7 +2702,7 @@ static void *__io_uaddr_map(struct page ***pages, unsigned short *npages,
        struct page **page_array;
        unsigned int nr_pages;
        void *page_addr;
-       int ret, i;
+       int ret, i, pinned;
 
        *npages = 0;
 
@@ -2716,12 +2716,12 @@ static void *__io_uaddr_map(struct page ***pages, unsigned short *npages,
        if (!page_array)
                return ERR_PTR(-ENOMEM);
 
-       ret = pin_user_pages_fast(uaddr, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
-                                       page_array);
-       if (ret != nr_pages) {
-err:
-               io_pages_free(&page_array, ret > 0 ? ret : 0);
-               return ret < 0 ? ERR_PTR(ret) : ERR_PTR(-EFAULT);
+
+       pinned = pin_user_pages_fast(uaddr, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
+                                    page_array);
+       if (pinned != nr_pages) {
+               ret = (pinned < 0) ? pinned : -EFAULT;
+               goto free_pages;
        }
 
        page_addr = page_address(page_array[0]);
@@ -2735,7 +2735,7 @@ static void *__io_uaddr_map(struct page ***pages, unsigned short *npages,
                 * didn't support this feature.
                 */
                if (PageHighMem(page_array[i]))
-                       goto err;
+                       goto free_pages;
 
                /*
                 * No support for discontig pages for now, should either be a
@@ -2744,13 +2744,17 @@ static void *__io_uaddr_map(struct page ***pages, unsigned short *npages,
                 * just fail them with EINVAL.
                 */
                if (page_address(page_array[i]) != page_addr)
-                       goto err;
+                       goto free_pages;
                page_addr += PAGE_SIZE;
        }
 
        *pages = page_array;
        *npages = nr_pages;
        return page_to_virt(page_array[0]);
+
+free_pages:
+       io_pages_free(&page_array, pinned > 0 ? pinned : 0);
+       return ERR_PTR(ret);
 }
 
 static void *io_rings_map(struct io_ring_ctx *ctx, unsigned long uaddr,