erofs: switch compressed_pages[] to bufvec

Convert compressed_pages[] to bufvec in order to avoid using
page->private to keep onlinepage_index (decompressed offset)
for inplace I/O pages.

In the future, we only rely on folio->private to keep a countdown
to unlock folios and set folio_uptodate.

Acked-by: Chao Yu <chao@kernel.org>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Link: https://lore.kernel.org/r/20220715154203.48093-8-hsiangkao@linux.alibaba.com

diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
index 423d4daf7ed9a2b9c2ce13955fc40a6ae192665a..2ea8c97be5b6da8ae726fa022c7dff7b50433c41 100644 (file)
--- a/fs/erofs/zdata.c
+++ b/fs/erofs/zdata.c
@@ -134,7 +134,7 @@ static int z_erofs_create_pcluster_pool(void)
 
        for (pcs = pcluster_pool;
             pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) {
-               size = struct_size(a, compressed_pages, pcs->maxpages);
+               size = struct_size(a, compressed_bvecs, pcs->maxpages);
 
                sprintf(pcs->name, "erofs_pcluster-%u", pcs->maxpages);
                pcs->slab = kmem_cache_create(pcs->name, size, 0,
@@ -287,16 +287,16 @@ struct z_erofs_decompress_frontend {
 
        struct page *candidate_bvpage;
        struct z_erofs_pcluster *pcl, *tailpcl;
-       /* a pointer used to pick up inplace I/O pages */
-       struct page **icpage_ptr;
        z_erofs_next_pcluster_t owned_head;
-
        enum z_erofs_collectmode mode;
 
        bool readahead;
        /* used for applying cache strategy on the fly */
        bool backmost;
        erofs_off_t headoffset;
+
+       /* a pointer used to pick up inplace I/O pages */
+       unsigned int icur;
 };
 
 #define DECOMPRESS_FRONTEND_INIT(__i) { \
@@ -319,24 +319,21 @@ static void z_erofs_bind_cache(struct z_erofs_decompress_frontend *fe,
         */
        gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) |
                        __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
-       struct page **pages;
-       pgoff_t index;
+       unsigned int i;
 
        if (fe->mode < COLLECT_PRIMARY_FOLLOWED)
                return;
 
-       pages = pcl->compressed_pages;
-       index = pcl->obj.index;
-       for (; index < pcl->obj.index + pcl->pclusterpages; ++index, ++pages) {
+       for (i = 0; i < pcl->pclusterpages; ++i) {
                struct page *page;
                compressed_page_t t;
                struct page *newpage = NULL;
 
                /* the compressed page was loaded before */
-               if (READ_ONCE(*pages))
+               if (READ_ONCE(pcl->compressed_bvecs[i].page))
                        continue;
 
-               page = find_get_page(mc, index);
+               page = find_get_page(mc, pcl->obj.index + i);
 
                if (page) {
                        t = tag_compressed_page_justfound(page);
@@ -357,7 +354,8 @@ static void z_erofs_bind_cache(struct z_erofs_decompress_frontend *fe,
                        }
                }
 
-               if (!cmpxchg_relaxed(pages, NULL, tagptr_cast_ptr(t)))
+               if (!cmpxchg_relaxed(&pcl->compressed_bvecs[i].page, NULL,
+                                    tagptr_cast_ptr(t)))
                        continue;
 
                if (page)
@@ -388,7 +386,7 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
         * therefore no need to worry about available decompression users.
         */
        for (i = 0; i < pcl->pclusterpages; ++i) {
-               struct page *page = pcl->compressed_pages[i];
+               struct page *page = pcl->compressed_bvecs[i].page;
 
                if (!page)
                        continue;
@@ -401,7 +399,7 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
                        continue;
 
                /* barrier is implied in the following 'unlock_page' */
-               WRITE_ONCE(pcl->compressed_pages[i], NULL);
+               WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL);
                detach_page_private(page);
                unlock_page(page);
        }
@@ -411,36 +409,39 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
 int erofs_try_to_free_cached_page(struct page *page)
 {
        struct z_erofs_pcluster *const pcl = (void *)page_private(page);
-       int ret = 0;    /* 0 - busy */
+       int ret, i;
 
-       if (erofs_workgroup_try_to_freeze(&pcl->obj, 1)) {
-               unsigned int i;
+       if (!erofs_workgroup_try_to_freeze(&pcl->obj, 1))
+               return 0;
 
-               DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
-               for (i = 0; i < pcl->pclusterpages; ++i) {
-                       if (pcl->compressed_pages[i] == page) {
-                               WRITE_ONCE(pcl->compressed_pages[i], NULL);
-                               ret = 1;
-                               break;
-                       }
+       ret = 0;
+       DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
+       for (i = 0; i < pcl->pclusterpages; ++i) {
+               if (pcl->compressed_bvecs[i].page == page) {
+                       WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL);
+                       ret = 1;
+                       break;
                }
-               erofs_workgroup_unfreeze(&pcl->obj, 1);
-
-               if (ret)
-                       detach_page_private(page);
        }
+       erofs_workgroup_unfreeze(&pcl->obj, 1);
+       if (ret)
+               detach_page_private(page);
        return ret;
 }
 
 /* page_type must be Z_EROFS_PAGE_TYPE_EXCLUSIVE */
 static bool z_erofs_try_inplace_io(struct z_erofs_decompress_frontend *fe,
-                                  struct page *page)
+                                  struct z_erofs_bvec *bvec)
 {
        struct z_erofs_pcluster *const pcl = fe->pcl;
 
-       while (fe->icpage_ptr > pcl->compressed_pages)
-               if (!cmpxchg(--fe->icpage_ptr, NULL, page))
+       while (fe->icur > 0) {
+               if (!cmpxchg(&pcl->compressed_bvecs[--fe->icur].page,
+                            NULL, bvec->page)) {
+                       pcl->compressed_bvecs[fe->icur] = *bvec;
                        return true;
+               }
+       }
        return false;
 }
 
@@ -454,7 +455,7 @@ static int z_erofs_attach_page(struct z_erofs_decompress_frontend *fe,
        if (fe->mode >= COLLECT_PRIMARY &&
            type == Z_EROFS_PAGE_TYPE_EXCLUSIVE) {
                /* give priority for inplaceio to use file pages first */
-               if (z_erofs_try_inplace_io(fe, bvec->page))
+               if (z_erofs_try_inplace_io(fe, bvec))
                        return 0;
                /* otherwise, check if it can be used as a bvpage */
                if (fe->mode >= COLLECT_PRIMARY_FOLLOWED &&
@@ -648,8 +649,7 @@ static int z_erofs_collector_begin(struct z_erofs_decompress_frontend *fe)
        z_erofs_bvec_iter_begin(&fe->biter, &fe->pcl->bvset,
                                Z_EROFS_INLINE_BVECS, fe->pcl->vcnt);
        /* since file-backed online pages are traversed in reverse order */
-       fe->icpage_ptr = fe->pcl->compressed_pages +
-                       z_erofs_pclusterpages(fe->pcl);
+       fe->icur = z_erofs_pclusterpages(fe->pcl);
        return 0;
 }
 
@@ -769,7 +769,8 @@ repeat:
                        goto err_out;
                }
                get_page(fe->map.buf.page);
-               WRITE_ONCE(fe->pcl->compressed_pages[0], fe->map.buf.page);
+               WRITE_ONCE(fe->pcl->compressed_bvecs[0].page,
+                          fe->map.buf.page);
                fe->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE;
        } else {
                /* bind cache first when cached decompression is preferred */
@@ -927,8 +928,9 @@ static struct page **z_erofs_parse_in_bvecs(struct erofs_sb_info *sbi,
        *overlapped = false;
 
        for (i = 0; i < pclusterpages; ++i) {
-               unsigned int pagenr;
-               struct page *page = pcl->compressed_pages[i];
+               struct z_erofs_bvec *bvec = &pcl->compressed_bvecs[i];
+               struct page *page = bvec->page;
+               unsigned int pgnr;
 
                /* compressed pages ought to be present before decompressing */
                if (!page) {
@@ -951,21 +953,15 @@ static struct page **z_erofs_parse_in_bvecs(struct erofs_sb_info *sbi,
                                continue;
                        }
 
-                       /*
-                        * only if non-head page can be selected
-                        * for inplace decompression
-                        */
-                       pagenr = z_erofs_onlinepage_index(page);
-
-                       DBG_BUGON(pagenr >= pcl->nr_pages);
-                       if (pages[pagenr]) {
+                       pgnr = (bvec->offset + pcl->pageofs_out) >> PAGE_SHIFT;
+                       DBG_BUGON(pgnr >= pcl->nr_pages);
+                       if (pages[pgnr]) {
                                DBG_BUGON(1);
-                               SetPageError(pages[pagenr]);
-                               z_erofs_onlinepage_endio(pages[pagenr]);
+                               SetPageError(pages[pgnr]);
+                               z_erofs_onlinepage_endio(pages[pgnr]);
                                err = -EFSCORRUPTED;
                        }
-                       pages[pagenr] = page;
-
+                       pages[pgnr] = page;
                        *overlapped = true;
                }
 
@@ -1067,19 +1063,19 @@ static int z_erofs_decompress_pcluster(struct super_block *sb,
 out:
        /* must handle all compressed pages before actual file pages */
        if (z_erofs_is_inline_pcluster(pcl)) {
-               page = pcl->compressed_pages[0];
-               WRITE_ONCE(pcl->compressed_pages[0], NULL);
+               page = pcl->compressed_bvecs[0].page;
+               WRITE_ONCE(pcl->compressed_bvecs[0].page, NULL);
                put_page(page);
        } else {
                for (i = 0; i < pclusterpages; ++i) {
-                       page = pcl->compressed_pages[i];
+                       page = pcl->compressed_bvecs[i].page;
 
                        if (erofs_page_is_managed(sbi, page))
                                continue;
 
                        /* recycle all individual short-lived pages */
                        (void)z_erofs_put_shortlivedpage(pagepool, page);
-                       WRITE_ONCE(pcl->compressed_pages[i], NULL);
+                       WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL);
                }
        }
        kfree(compressed_pages);
@@ -1193,7 +1189,7 @@ static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl,
        int justfound;
 
 repeat:
-       page = READ_ONCE(pcl->compressed_pages[nr]);
+       page = READ_ONCE(pcl->compressed_bvecs[nr].page);
        oldpage = page;
 
        if (!page)
@@ -1209,7 +1205,7 @@ repeat:
         * otherwise, it will go inplace I/O path instead.
         */
        if (page->private == Z_EROFS_PREALLOCATED_PAGE) {
-               WRITE_ONCE(pcl->compressed_pages[nr], page);
+               WRITE_ONCE(pcl->compressed_bvecs[nr].page, page);
                set_page_private(page, 0);
                tocache = true;
                goto out_tocache;
@@ -1235,14 +1231,14 @@ repeat:
 
        /* the page is still in manage cache */
        if (page->mapping == mc) {
-               WRITE_ONCE(pcl->compressed_pages[nr], page);
+               WRITE_ONCE(pcl->compressed_bvecs[nr].page, page);
 
                ClearPageError(page);
                if (!PagePrivate(page)) {
                        /*
                         * impossible to be !PagePrivate(page) for
                         * the current restriction as well if
-                        * the page is already in compressed_pages[].
+                        * the page is already in compressed_bvecs[].
                         */
                        DBG_BUGON(!justfound);
 
@@ -1271,7 +1267,8 @@ repeat:
        put_page(page);
 out_allocpage:
        page = erofs_allocpage(pagepool, gfp | __GFP_NOFAIL);
-       if (oldpage != cmpxchg(&pcl->compressed_pages[nr], oldpage, page)) {
+       if (oldpage != cmpxchg(&pcl->compressed_bvecs[nr].page,
+                              oldpage, page)) {
                erofs_pagepool_add(pagepool, page);
                cond_resched();
                goto repeat;

diff --git a/fs/erofs/zdata.h b/fs/erofs/zdata.h
index 468f6308fc90e172b8644a3f00fff2ecc1e5a595..5d236c8b40c5d9ab9b801fe5bb96efb594798558 100644 (file)
--- a/fs/erofs/zdata.h
+++ b/fs/erofs/zdata.h
@@ -87,8 +87,8 @@ struct z_erofs_pcluster {
        /* I: compression algorithm format */
        unsigned char algorithmformat;
 
-       /* A: compressed pages (can be cached or inplaced pages) */
-       struct page *compressed_pages[];
+       /* A: compressed bvecs (can be cached or inplaced pages) */
+       struct z_erofs_bvec compressed_bvecs[];
 };
 
 /* let's avoid the valid 32-bit kernel addresses */