^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2018 HUAWEI, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * https://www.huawei.com/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Created by Gao Xiang <gaoxiang25@huawei.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #ifndef __EROFS_FS_ZDATA_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #define __EROFS_FS_ZDATA_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include "zpvec.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #define Z_EROFS_PCLUSTER_MAX_PAGES (Z_EROFS_PCLUSTER_MAX_SIZE / PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #define Z_EROFS_NR_INLINE_PAGEVECS 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * Structure fields follow one of the following exclusion rules.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * I: Modifiable by initialization/destruction paths and read-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * for everyone else;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * L: Field should be protected by pageset lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * A: Field should be accessed / updated in atomic for parallelized code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) struct z_erofs_collection {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) struct mutex lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /* I: page offset of start position of decompression */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) unsigned short pageofs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) /* L: maximum relative page index in pagevec[] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) unsigned short nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) /* L: total number of pages in pagevec[] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) unsigned int vcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /* L: inline a certain number of pagevecs for bootstrap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) erofs_vtptr_t pagevec[Z_EROFS_NR_INLINE_PAGEVECS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /* I: can be used to free the pcluster by RCU. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct rcu_head rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define Z_EROFS_PCLUSTER_FULL_LENGTH 0x00000001
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define Z_EROFS_PCLUSTER_LENGTH_BIT 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * let's leave a type here in case of introducing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * another tagged pointer later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) typedef void *z_erofs_next_pcluster_t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct z_erofs_pcluster {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) struct erofs_workgroup obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) struct z_erofs_collection primary_collection;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /* A: point to next chained pcluster or TAILs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) z_erofs_next_pcluster_t next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /* A: lower limit of decompressed length and if full length or not */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) unsigned int length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) /* I: physical cluster size in pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) unsigned short pclusterpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /* I: compression algorithm format */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) unsigned char algorithmformat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) /* A: compressed pages (can be cached or inplaced pages) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct page *compressed_pages[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define z_erofs_primarycollection(pcluster) (&(pcluster)->primary_collection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) /* let's avoid the valid 32-bit kernel addresses */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /* the chained workgroup has't submitted io (still open) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #define Z_EROFS_PCLUSTER_TAIL ((void *)0x5F0ECAFE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) /* the chained workgroup has already submitted io */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define Z_EROFS_PCLUSTER_TAIL_CLOSED ((void *)0x5F0EDEAD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define Z_EROFS_PCLUSTER_NIL (NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct z_erofs_decompressqueue {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct super_block *sb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) atomic_t pending_bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) z_erofs_next_pcluster_t head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) wait_queue_head_t wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) struct work_struct work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) } u;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #define MNGD_MAPPING(sbi) ((sbi)->managed_cache->i_mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static inline bool erofs_page_is_managed(const struct erofs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return page->mapping == MNGD_MAPPING(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #define Z_EROFS_ONLINEPAGE_COUNT_BITS 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define Z_EROFS_ONLINEPAGE_COUNT_MASK ((1 << Z_EROFS_ONLINEPAGE_COUNT_BITS) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) #define Z_EROFS_ONLINEPAGE_INDEX_SHIFT (Z_EROFS_ONLINEPAGE_COUNT_BITS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * waiters (aka. ongoing_packs): # to unlock the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * sub-index: 0 - for partial page, >= 1 full page sub-index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) typedef atomic_t z_erofs_onlinepage_t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /* type punning */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) union z_erofs_onlinepage_converter {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) z_erofs_onlinepage_t *o;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) unsigned long *v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static inline unsigned int z_erofs_onlinepage_index(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) union z_erofs_onlinepage_converter u;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) DBG_BUGON(!PagePrivate(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) u.v = &page_private(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) return atomic_read(u.o) >> Z_EROFS_ONLINEPAGE_INDEX_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) static inline void z_erofs_onlinepage_init(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) z_erofs_onlinepage_t o;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) unsigned long v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /* keep from being unlocked in advance */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) } u = { .o = ATOMIC_INIT(1) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) set_page_private(page, u.v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) SetPagePrivate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) static inline void z_erofs_onlinepage_fixup(struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) uintptr_t index, bool down)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) union z_erofs_onlinepage_converter u = { .v = &page_private(page) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) int orig, orig_index, val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) repeat:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) orig = atomic_read(u.o);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) orig_index = orig >> Z_EROFS_ONLINEPAGE_INDEX_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) if (orig_index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (!index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) DBG_BUGON(orig_index != index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) val = (index << Z_EROFS_ONLINEPAGE_INDEX_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) ((orig & Z_EROFS_ONLINEPAGE_COUNT_MASK) + (unsigned int)down);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) if (atomic_cmpxchg(u.o, orig, val) != orig)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) goto repeat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static inline void z_erofs_onlinepage_endio(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) union z_erofs_onlinepage_converter u;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) unsigned int v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) DBG_BUGON(!PagePrivate(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) u.v = &page_private(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) v = atomic_dec_return(u.o);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (!(v & Z_EROFS_ONLINEPAGE_COUNT_MASK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) set_page_private(page, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) ClearPagePrivate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (!PageError(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) SetPageUptodate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) erofs_dbg("%s, page %p value %x", __func__, page, atomic_read(u.o));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) #define Z_EROFS_VMAP_ONSTACK_PAGES \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) min_t(unsigned int, THREAD_SIZE / 8 / sizeof(struct page *), 96U)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) #define Z_EROFS_VMAP_GLOBAL_PAGES 2048
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags