^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (c) 2016-present, Facebook, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/bio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/bitmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/refcount.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/zstd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "misc.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "compression.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "ctree.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define ZSTD_BTRFS_MAX_WINDOWLOG 17
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define ZSTD_BTRFS_DEFAULT_LEVEL 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define ZSTD_BTRFS_MAX_LEVEL 15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /* 307s to avoid pathologically clashing with transaction commit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define ZSTD_BTRFS_RECLAIM_JIFFIES (307 * HZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) static ZSTD_parameters zstd_get_btrfs_parameters(unsigned int level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) size_t src_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) ZSTD_parameters params = ZSTD_getParams(level, src_len, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) return params;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) struct workspace {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) void *mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) char *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) unsigned int level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) unsigned int req_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) unsigned long last_used; /* jiffies */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct list_head lru_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) ZSTD_inBuffer in_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) ZSTD_outBuffer out_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * Zstd Workspace Management
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * Zstd workspaces have different memory requirements depending on the level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * The zstd workspaces are managed by having individual lists for each level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * and a global lru. Forward progress is maintained by protecting a max level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * workspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * Getting a workspace is done by using the bitmap to identify the levels that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * have available workspaces and scans up. This lets us recycle higher level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * workspaces because of the monotonic memory guarantee. A workspace's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * last_used is only updated if it is being used by the corresponding memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * level. Putting a workspace involves adding it back to the appropriate places
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * and adding it back to the lru if necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * A timer is used to reclaim workspaces if they have not been used for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * ZSTD_BTRFS_RECLAIM_JIFFIES. This helps keep only active workspaces around.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * The upper bound is provided by the workqueue limit which is 2 (percpu limit).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct zstd_workspace_manager {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) const struct btrfs_compress_op *ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) struct list_head lru_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct list_head idle_ws[ZSTD_BTRFS_MAX_LEVEL];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) unsigned long active_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) wait_queue_head_t wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct timer_list timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static struct zstd_workspace_manager wsm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static size_t zstd_ws_mem_sizes[ZSTD_BTRFS_MAX_LEVEL];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static inline struct workspace *list_to_workspace(struct list_head *list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) return container_of(list, struct workspace, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) void zstd_free_workspace(struct list_head *ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) struct list_head *zstd_alloc_workspace(unsigned int level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * zstd_reclaim_timer_fn - reclaim timer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * @t: timer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * This scans the lru_list and attempts to reclaim any workspace that hasn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * been used for ZSTD_BTRFS_RECLAIM_JIFFIES.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static void zstd_reclaim_timer_fn(struct timer_list *timer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) unsigned long reclaim_threshold = jiffies - ZSTD_BTRFS_RECLAIM_JIFFIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) struct list_head *pos, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) spin_lock_bh(&wsm.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) if (list_empty(&wsm.lru_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) spin_unlock_bh(&wsm.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) list_for_each_prev_safe(pos, next, &wsm.lru_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) struct workspace *victim = container_of(pos, struct workspace,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) lru_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) unsigned int level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (time_after(victim->last_used, reclaim_threshold))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /* workspace is in use */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (victim->req_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) level = victim->level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) list_del(&victim->lru_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) list_del(&victim->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) zstd_free_workspace(&victim->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (list_empty(&wsm.idle_ws[level - 1]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) clear_bit(level - 1, &wsm.active_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) if (!list_empty(&wsm.lru_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) mod_timer(&wsm.timer, jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) spin_unlock_bh(&wsm.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * zstd_calc_ws_mem_sizes - calculate monotonic memory bounds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * It is possible based on the level configurations that a higher level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * workspace uses less memory than a lower level workspace. In order to reuse
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * workspaces, this must be made a monotonic relationship. This precomputes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * the required memory for each level and enforces the monotonicity between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * level and memory required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) static void zstd_calc_ws_mem_sizes(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) size_t max_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) unsigned int level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) for (level = 1; level <= ZSTD_BTRFS_MAX_LEVEL; level++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) ZSTD_parameters params =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) zstd_get_btrfs_parameters(level, ZSTD_BTRFS_MAX_INPUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) size_t level_size =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) max_t(size_t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) ZSTD_CStreamWorkspaceBound(params.cParams),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) max_size = max_t(size_t, max_size, level_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) zstd_ws_mem_sizes[level - 1] = max_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) void zstd_init_workspace_manager(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) struct list_head *ws;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) zstd_calc_ws_mem_sizes();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) wsm.ops = &btrfs_zstd_compress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) spin_lock_init(&wsm.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) init_waitqueue_head(&wsm.wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) timer_setup(&wsm.timer, zstd_reclaim_timer_fn, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) INIT_LIST_HEAD(&wsm.lru_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) for (i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) INIT_LIST_HEAD(&wsm.idle_ws[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) ws = zstd_alloc_workspace(ZSTD_BTRFS_MAX_LEVEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) if (IS_ERR(ws)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) pr_warn(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) "BTRFS: cannot preallocate zstd compression workspace\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) set_bit(ZSTD_BTRFS_MAX_LEVEL - 1, &wsm.active_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) list_add(ws, &wsm.idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) void zstd_cleanup_workspace_manager(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct workspace *workspace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) spin_lock_bh(&wsm.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) for (i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) while (!list_empty(&wsm.idle_ws[i])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) workspace = container_of(wsm.idle_ws[i].next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) struct workspace, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) list_del(&workspace->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) list_del(&workspace->lru_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) zstd_free_workspace(&workspace->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) spin_unlock_bh(&wsm.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) del_timer_sync(&wsm.timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * zstd_find_workspace - find workspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * @level: compression level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * This iterates over the set bits in the active_map beginning at the requested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * compression level. This lets us utilize already allocated workspaces before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * allocating a new one. If the workspace is of a larger size, it is used, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * the place in the lru_list and last_used times are not updated. This is to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * offer the opportunity to reclaim the workspace in favor of allocating an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * appropriately sized one in the future.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static struct list_head *zstd_find_workspace(unsigned int level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) struct list_head *ws;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct workspace *workspace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) int i = level - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) spin_lock_bh(&wsm.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) for_each_set_bit_from(i, &wsm.active_map, ZSTD_BTRFS_MAX_LEVEL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (!list_empty(&wsm.idle_ws[i])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) ws = wsm.idle_ws[i].next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) workspace = list_to_workspace(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) list_del_init(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) /* keep its place if it's a lower level using this */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) workspace->req_level = level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) if (level == workspace->level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) list_del(&workspace->lru_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (list_empty(&wsm.idle_ws[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) clear_bit(i, &wsm.active_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) spin_unlock_bh(&wsm.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) return ws;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) spin_unlock_bh(&wsm.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * zstd_get_workspace - zstd's get_workspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) * @level: compression level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * If @level is 0, then any compression level can be used. Therefore, we begin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * scanning from 1. We first scan through possible workspaces and then after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * attempt to allocate a new workspace. If we fail to allocate one due to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * memory pressure, go to sleep waiting for the max level workspace to free up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) struct list_head *zstd_get_workspace(unsigned int level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) struct list_head *ws;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) unsigned int nofs_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) /* level == 0 means we can use any workspace */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) if (!level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) level = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) ws = zstd_find_workspace(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return ws;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) nofs_flag = memalloc_nofs_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) ws = zstd_alloc_workspace(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) memalloc_nofs_restore(nofs_flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (IS_ERR(ws)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) DEFINE_WAIT(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) prepare_to_wait(&wsm.wait, &wait, TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) finish_wait(&wsm.wait, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return ws;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * zstd_put_workspace - zstd put_workspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * @ws: list_head for the workspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * When putting back a workspace, we only need to update the LRU if we are of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * the requested compression level. Here is where we continue to protect the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * max level workspace or update last_used accordingly. If the reclaim timer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * isn't set, it is also set here. Only the max level workspace tries and wakes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * up waiting workspaces.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) void zstd_put_workspace(struct list_head *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) struct workspace *workspace = list_to_workspace(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) spin_lock_bh(&wsm.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) /* A node is only taken off the lru if we are the corresponding level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (workspace->req_level == workspace->level) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /* Hide a max level workspace from reclaim */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (list_empty(&wsm.idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) INIT_LIST_HEAD(&workspace->lru_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) workspace->last_used = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) list_add(&workspace->lru_list, &wsm.lru_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (!timer_pending(&wsm.timer))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) mod_timer(&wsm.timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) set_bit(workspace->level - 1, &wsm.active_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) list_add(&workspace->list, &wsm.idle_ws[workspace->level - 1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) workspace->req_level = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) spin_unlock_bh(&wsm.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (workspace->level == ZSTD_BTRFS_MAX_LEVEL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) cond_wake_up(&wsm.wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) void zstd_free_workspace(struct list_head *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) struct workspace *workspace = list_entry(ws, struct workspace, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) kvfree(workspace->mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) kfree(workspace->buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) kfree(workspace);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) struct list_head *zstd_alloc_workspace(unsigned int level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct workspace *workspace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) if (!workspace)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) workspace->size = zstd_ws_mem_sizes[level - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) workspace->level = level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) workspace->req_level = level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) workspace->last_used = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) workspace->mem = kvmalloc(workspace->size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (!workspace->mem || !workspace->buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) INIT_LIST_HEAD(&workspace->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) INIT_LIST_HEAD(&workspace->lru_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) return &workspace->list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) zstd_free_workspace(&workspace->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) int zstd_compress_pages(struct list_head *ws, struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) u64 start, struct page **pages, unsigned long *out_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) unsigned long *total_in, unsigned long *total_out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) struct workspace *workspace = list_entry(ws, struct workspace, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) ZSTD_CStream *stream;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) int nr_pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) struct page *in_page = NULL; /* The current page to read */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) struct page *out_page = NULL; /* The current page to write to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) unsigned long tot_in = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) unsigned long tot_out = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) unsigned long len = *total_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) const unsigned long nr_dest_pages = *out_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) unsigned long max_out = nr_dest_pages * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) ZSTD_parameters params = zstd_get_btrfs_parameters(workspace->req_level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) *out_pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) *total_out = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) *total_in = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) /* Initialize the stream */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) stream = ZSTD_initCStream(params, len, workspace->mem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) workspace->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) if (!stream) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) pr_warn("BTRFS: ZSTD_initCStream failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) /* map in the first page of input data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) in_page = find_get_page(mapping, start >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) workspace->in_buf.src = kmap(in_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) workspace->in_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) /* Allocate and map in the output buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) if (out_page == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) pages[nr_pages++] = out_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) workspace->out_buf.dst = kmap(out_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) workspace->out_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) size_t ret2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) ret2 = ZSTD_compressStream(stream, &workspace->out_buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) &workspace->in_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) if (ZSTD_isError(ret2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) pr_debug("BTRFS: ZSTD_compressStream returned %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) ZSTD_getErrorCode(ret2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) /* Check to see if we are making it bigger */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (tot_in + workspace->in_buf.pos > 8192 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) tot_in + workspace->in_buf.pos <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) tot_out + workspace->out_buf.pos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) ret = -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /* We've reached the end of our output range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (workspace->out_buf.pos >= max_out) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) tot_out += workspace->out_buf.pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) ret = -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) /* Check if we need more output space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) if (workspace->out_buf.pos == workspace->out_buf.size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) tot_out += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) max_out -= PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) kunmap(out_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) if (nr_pages == nr_dest_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) out_page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) ret = -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (out_page == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) pages[nr_pages++] = out_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) workspace->out_buf.dst = kmap(out_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) workspace->out_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) workspace->out_buf.size = min_t(size_t, max_out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) /* We've reached the end of the input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (workspace->in_buf.pos >= len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) tot_in += workspace->in_buf.pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) /* Check if we need more input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if (workspace->in_buf.pos == workspace->in_buf.size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) tot_in += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) kunmap(in_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) put_page(in_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) start += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) len -= PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) in_page = find_get_page(mapping, start >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) workspace->in_buf.src = kmap(in_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) workspace->in_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) size_t ret2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) ret2 = ZSTD_endStream(stream, &workspace->out_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (ZSTD_isError(ret2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) pr_debug("BTRFS: ZSTD_endStream returned %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) ZSTD_getErrorCode(ret2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (ret2 == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) tot_out += workspace->out_buf.pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (workspace->out_buf.pos >= max_out) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) tot_out += workspace->out_buf.pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) ret = -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) tot_out += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) max_out -= PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) kunmap(out_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) if (nr_pages == nr_dest_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) out_page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) ret = -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (out_page == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) pages[nr_pages++] = out_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) workspace->out_buf.dst = kmap(out_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) workspace->out_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) if (tot_out >= tot_in) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) ret = -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) *total_in = tot_in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) *total_out = tot_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) *out_pages = nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) /* Cleanup */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) if (in_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) kunmap(in_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) put_page(in_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) if (out_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) kunmap(out_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) struct workspace *workspace = list_entry(ws, struct workspace, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) struct page **pages_in = cb->compressed_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) u64 disk_start = cb->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) struct bio *orig_bio = cb->orig_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) size_t srclen = cb->compressed_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) ZSTD_DStream *stream;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) unsigned long page_in_index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) unsigned long buf_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) unsigned long total_out = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) stream = ZSTD_initDStream(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (!stream) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) pr_debug("BTRFS: ZSTD_initDStream failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) workspace->in_buf.src = kmap(pages_in[page_in_index]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) workspace->in_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) workspace->out_buf.dst = workspace->buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) workspace->out_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) workspace->out_buf.size = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) size_t ret2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) ret2 = ZSTD_decompressStream(stream, &workspace->out_buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) &workspace->in_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) if (ZSTD_isError(ret2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) ZSTD_getErrorCode(ret2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) buf_start = total_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) total_out += workspace->out_buf.pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) workspace->out_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) ret = btrfs_decompress_buf2page(workspace->out_buf.dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) buf_start, total_out, disk_start, orig_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) if (workspace->in_buf.pos >= srclen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) /* Check if we've hit the end of a frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (ret2 == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) if (workspace->in_buf.pos == workspace->in_buf.size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) kunmap(pages_in[page_in_index++]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) if (page_in_index >= total_pages_in) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) workspace->in_buf.src = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) srclen -= PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) workspace->in_buf.src = kmap(pages_in[page_in_index]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) workspace->in_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) zero_fill_bio(orig_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) if (workspace->in_buf.src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) kunmap(pages_in[page_in_index]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) int zstd_decompress(struct list_head *ws, unsigned char *data_in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) struct page *dest_page, unsigned long start_byte, size_t srclen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) size_t destlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) struct workspace *workspace = list_entry(ws, struct workspace, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) ZSTD_DStream *stream;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) size_t ret2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) unsigned long total_out = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) unsigned long pg_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) char *kaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) stream = ZSTD_initDStream(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) if (!stream) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) pr_warn("BTRFS: ZSTD_initDStream failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) goto finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) destlen = min_t(size_t, destlen, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) workspace->in_buf.src = data_in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) workspace->in_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) workspace->in_buf.size = srclen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) workspace->out_buf.dst = workspace->buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) workspace->out_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) workspace->out_buf.size = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) ret2 = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) while (pg_offset < destlen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) && workspace->in_buf.pos < workspace->in_buf.size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) unsigned long buf_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) unsigned long buf_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) unsigned long bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) /* Check if the frame is over and we still need more input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) if (ret2 == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) pr_debug("BTRFS: ZSTD_decompressStream ended early\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) goto finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) ret2 = ZSTD_decompressStream(stream, &workspace->out_buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) &workspace->in_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) if (ZSTD_isError(ret2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) ZSTD_getErrorCode(ret2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) goto finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) buf_start = total_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) total_out += workspace->out_buf.pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) workspace->out_buf.pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if (total_out <= start_byte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) if (total_out > start_byte && buf_start < start_byte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) buf_offset = start_byte - buf_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) buf_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) bytes = min_t(unsigned long, destlen - pg_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) workspace->out_buf.size - buf_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) kaddr = kmap_atomic(dest_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) memcpy(kaddr + pg_offset, workspace->out_buf.dst + buf_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) kunmap_atomic(kaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) pg_offset += bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) finish:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (pg_offset < destlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) kaddr = kmap_atomic(dest_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) memset(kaddr + pg_offset, 0, destlen - pg_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) kunmap_atomic(kaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) const struct btrfs_compress_op btrfs_zstd_compress = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) /* ZSTD uses own workspace manager */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) .workspace_manager = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) .max_level = ZSTD_BTRFS_MAX_LEVEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) .default_level = ZSTD_BTRFS_DEFAULT_LEVEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) };
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags