^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright (C) 2002 Sistina Software (UK) Limited.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2006 Red Hat GmbH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This file is released under the GPL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Kcopyd provides a simple interface for copying an area of one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * block-device to one or more other block-devices, with an asynchronous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * completion notification.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/mempool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/of_platform.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/of_reserved_mem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/device-mapper.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/dm-kcopyd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include "dm-core.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define SPLIT_COUNT 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define MIN_JOBS 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define DEFAULT_SUB_JOB_SIZE_KB 512
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define MAX_SUB_JOB_SIZE_KB 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static unsigned kcopyd_subjob_size_kb = DEFAULT_SUB_JOB_SIZE_KB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) module_param(kcopyd_subjob_size_kb, uint, S_IRUGO | S_IWUSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) MODULE_PARM_DESC(kcopyd_subjob_size_kb, "Sub-job size for dm-kcopyd clients");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static bool rsm_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) static phys_addr_t rsm_mem_base, rsm_mem_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #ifndef MODULE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static DEFINE_SPINLOCK(rsm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) static int *rsm_mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static int rsm_page_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static int rsm_tbl_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) static struct reserved_mem *rmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) static void __init kcopyd_rsm_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) static struct device_node *rsm_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) if (!rsm_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) rsm_node = of_find_compatible_node(NULL, NULL, "mediatek,dm_ota");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) if (!rsm_node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) rmem = of_reserved_mem_lookup(rsm_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) if (!rmem) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) goto out_put_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) rsm_mem_base = rmem->base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) rsm_mem_size = rmem->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) rsm_page_cnt = rsm_mem_size / PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) rsm_mem = kcalloc(rsm_page_cnt, sizeof(int), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (!rsm_mem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) out_put_node:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) of_node_put(rsm_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) pr_warn("kcopyd: failed to init rsm: %d", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static int __init kcopyd_rsm_enable(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) rsm_enabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) early_param("mtk_kcopyd_quirk", kcopyd_rsm_enable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static void kcopyd_rsm_get_page(struct page **p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) *p = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) spin_lock_irqsave(&rsm_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) for (i = 0 ; i < rsm_page_cnt ; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) rsm_tbl_idx = (rsm_tbl_idx + 1 == rsm_page_cnt) ? 0 : rsm_tbl_idx + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (rsm_mem[rsm_tbl_idx] == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) rsm_mem[rsm_tbl_idx] = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) *p = virt_to_page(phys_to_virt(rsm_mem_base + PAGE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * rsm_tbl_idx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) spin_unlock_irqrestore(&rsm_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) static void kcopyd_rsm_drop_page(struct page **p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) u64 off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (*p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) off = page_to_phys(*p) - rsm_mem_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) spin_lock_irqsave(&rsm_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) rsm_mem[off >> PAGE_SHIFT] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) spin_unlock_irqrestore(&rsm_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) *p = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) static void kcopyd_rsm_destroy(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (rsm_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) kfree(rsm_mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #define kcopyd_rsm_destroy(...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define kcopyd_rsm_drop_page(...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #define kcopyd_rsm_get_page(...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #define kcopyd_rsm_init(...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) static unsigned dm_get_kcopyd_subjob_size(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) unsigned sub_job_size_kb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) sub_job_size_kb = __dm_get_module_param(&kcopyd_subjob_size_kb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) DEFAULT_SUB_JOB_SIZE_KB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) MAX_SUB_JOB_SIZE_KB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return sub_job_size_kb << 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) /*-----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * Each kcopyd client has its own little pool of preallocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * pages for kcopyd io.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) *---------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct dm_kcopyd_client {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) struct page_list *pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) unsigned nr_reserved_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) unsigned nr_free_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) unsigned sub_job_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) struct dm_io_client *io_client;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) wait_queue_head_t destroyq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) mempool_t job_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) struct workqueue_struct *kcopyd_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) struct work_struct kcopyd_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) struct dm_kcopyd_throttle *throttle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) atomic_t nr_jobs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * We maintain four lists of jobs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * i) jobs waiting for pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * ii) jobs that have pages, and are waiting for the io to be issued.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * iii) jobs that don't need to do any IO and just run a callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * iv) jobs that have completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * All four of these are protected by job_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) spinlock_t job_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) struct list_head callback_jobs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) struct list_head complete_jobs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) struct list_head io_jobs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) struct list_head pages_jobs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) static struct page_list zero_page_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) static DEFINE_SPINLOCK(throttle_spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * IO/IDLE accounting slowly decays after (1 << ACCOUNT_INTERVAL_SHIFT) period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * When total_period >= (1 << ACCOUNT_INTERVAL_SHIFT) the counters are divided
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * by 2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #define ACCOUNT_INTERVAL_SHIFT SHIFT_HZ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * Sleep this number of milliseconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * The value was decided experimentally.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * Smaller values seem to cause an increased copy rate above the limit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * The reason for this is unknown but possibly due to jiffies rounding errors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * or read/write cache inside the disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #define SLEEP_MSEC 100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * Maximum number of sleep events. There is a theoretical livelock if more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * kcopyd clients do work simultaneously which this limit avoids.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) #define MAX_SLEEPS 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) static void io_job_start(struct dm_kcopyd_throttle *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) unsigned throttle, now, difference;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) int slept = 0, skew;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (unlikely(!t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) try_again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) spin_lock_irq(&throttle_spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) throttle = READ_ONCE(t->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (likely(throttle >= 100))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) goto skip_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) now = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) difference = now - t->last_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) t->last_jiffies = now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (t->num_io_jobs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) t->io_period += difference;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) t->total_period += difference;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * Maintain sane values if we got a temporary overflow.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if (unlikely(t->io_period > t->total_period))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) t->io_period = t->total_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (unlikely(t->total_period >= (1 << ACCOUNT_INTERVAL_SHIFT))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) int shift = fls(t->total_period >> ACCOUNT_INTERVAL_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) t->total_period >>= shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) t->io_period >>= shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) skew = t->io_period - throttle * t->total_period / 100;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (unlikely(skew > 0) && slept < MAX_SLEEPS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) slept++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) spin_unlock_irq(&throttle_spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) msleep(SLEEP_MSEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) goto try_again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) skip_limit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) t->num_io_jobs++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) spin_unlock_irq(&throttle_spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) static void io_job_finish(struct dm_kcopyd_throttle *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (unlikely(!t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) spin_lock_irqsave(&throttle_spinlock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) t->num_io_jobs--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) if (likely(READ_ONCE(t->throttle) >= 100))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) goto skip_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (!t->num_io_jobs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) unsigned now, difference;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) now = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) difference = now - t->last_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) t->last_jiffies = now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) t->io_period += difference;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) t->total_period += difference;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * Maintain sane values if we got a temporary overflow.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (unlikely(t->io_period > t->total_period))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) t->io_period = t->total_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) skip_limit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) spin_unlock_irqrestore(&throttle_spinlock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) static void wake(struct dm_kcopyd_client *kc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) queue_work(kc->kcopyd_wq, &kc->kcopyd_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * Obtain one page for the use of kcopyd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) static struct page_list *alloc_pl(gfp_t gfp, unsigned long job_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) struct page_list *pl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) pl = kmalloc(sizeof(*pl), gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (!pl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (rsm_enabled && test_bit(DM_KCOPYD_SNAP_MERGE, &job_flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) kcopyd_rsm_get_page(&pl->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) pl->page = alloc_page(gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (!pl->page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) kfree(pl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) return pl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) static void free_pl(struct page_list *pl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) struct page *p = pl->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) phys_addr_t pa = page_to_phys(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (rsm_enabled && pa >= rsm_mem_base && pa < rsm_mem_base + rsm_mem_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) kcopyd_rsm_drop_page(&pl->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) __free_page(pl->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) kfree(pl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * Add the provided pages to a client's free page list, releasing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) * back to the system any beyond the reserved_pages limit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) struct page_list *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) next = pl->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (kc->nr_free_pages >= kc->nr_reserved_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) free_pl(pl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) pl->next = kc->pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) kc->pages = pl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) kc->nr_free_pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) pl = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) } while (pl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) static int kcopyd_get_pages(struct dm_kcopyd_client *kc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) unsigned int nr, struct page_list **pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) unsigned long job_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) struct page_list *pl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) *pages = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) pl = alloc_pl(__GFP_NOWARN | __GFP_NORETRY | __GFP_KSWAPD_RECLAIM, job_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (unlikely(!pl)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) /* Use reserved pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) pl = kc->pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) if (unlikely(!pl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) goto out_of_memory;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) kc->pages = pl->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) kc->nr_free_pages--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) pl->next = *pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) *pages = pl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) } while (--nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) out_of_memory:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (*pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) kcopyd_put_pages(kc, *pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * These three functions resize the page pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static void drop_pages(struct page_list *pl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) struct page_list *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) while (pl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) next = pl->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) free_pl(pl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) pl = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * Allocate and reserve nr_pages for the use of a specific client.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) unsigned i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) struct page_list *pl = NULL, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) for (i = 0; i < nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) next = alloc_pl(GFP_KERNEL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (!next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (pl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) drop_pages(pl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) next->next = pl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) pl = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) kc->nr_reserved_pages += nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) kcopyd_put_pages(kc, pl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) static void client_free_pages(struct dm_kcopyd_client *kc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) BUG_ON(kc->nr_free_pages != kc->nr_reserved_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) drop_pages(kc->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) kc->pages = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) kc->nr_free_pages = kc->nr_reserved_pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) /*-----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * kcopyd_jobs need to be allocated by the *clients* of kcopyd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * for this reason we use a mempool to prevent the client from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) * ever having to do io (which could cause a deadlock).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) *---------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) struct kcopyd_job {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) struct dm_kcopyd_client *kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * Error state of the job.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) int read_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) unsigned long write_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * Either READ or WRITE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) int rw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) struct dm_io_region source;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * The destinations for the transfer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) unsigned int num_dests;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) struct dm_io_region dests[DM_KCOPYD_MAX_REGIONS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) struct page_list *pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * Set this to ensure you are notified when the job has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * completed. 'context' is for callback to use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) dm_kcopyd_notify_fn fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) void *context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * These fields are only used if the job has been split
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) * into more manageable parts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) struct mutex lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) atomic_t sub_jobs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) sector_t progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) sector_t write_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) struct kcopyd_job *master_job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) static struct kmem_cache *_job_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) int __init dm_kcopyd_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) _job_cache = kmem_cache_create("kcopyd_job",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) sizeof(struct kcopyd_job) * (SPLIT_COUNT + 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) __alignof__(struct kcopyd_job), 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (!_job_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) zero_page_list.next = &zero_page_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) zero_page_list.page = ZERO_PAGE(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) kcopyd_rsm_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) void dm_kcopyd_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) kmem_cache_destroy(_job_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) _job_cache = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) kcopyd_rsm_destroy();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * Functions to push and pop a job onto the head of a given job
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) static struct kcopyd_job *pop_io_job(struct list_head *jobs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) struct dm_kcopyd_client *kc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) struct kcopyd_job *job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) * For I/O jobs, pop any read, any write without sequential write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) * constraint and sequential writes that are at the right position.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) list_for_each_entry(job, jobs, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (job->rw == READ || !test_bit(DM_KCOPYD_WRITE_SEQ, &job->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) list_del(&job->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) return job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) if (job->write_offset == job->master_job->write_offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) job->master_job->write_offset += job->source.count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) list_del(&job->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) return job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) static struct kcopyd_job *pop(struct list_head *jobs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) struct dm_kcopyd_client *kc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) struct kcopyd_job *job = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) spin_lock_irqsave(&kc->job_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) if (!list_empty(jobs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (jobs == &kc->io_jobs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) job = pop_io_job(jobs, kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) job = list_entry(jobs->next, struct kcopyd_job, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) list_del(&job->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) spin_unlock_irqrestore(&kc->job_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) return job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) static void push(struct list_head *jobs, struct kcopyd_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) struct dm_kcopyd_client *kc = job->kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) spin_lock_irqsave(&kc->job_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) list_add_tail(&job->list, jobs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) spin_unlock_irqrestore(&kc->job_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) static void push_head(struct list_head *jobs, struct kcopyd_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) struct dm_kcopyd_client *kc = job->kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) spin_lock_irqsave(&kc->job_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) list_add(&job->list, jobs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) spin_unlock_irqrestore(&kc->job_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) * These three functions process 1 item from the corresponding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) * job list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) * They return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) * < 0: error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) * 0: success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) * > 0: can't process yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) static int run_complete_job(struct kcopyd_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) void *context = job->context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) int read_err = job->read_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) unsigned long write_err = job->write_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) dm_kcopyd_notify_fn fn = job->fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) struct dm_kcopyd_client *kc = job->kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) if (job->pages && job->pages != &zero_page_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) kcopyd_put_pages(kc, job->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) * If this is the master job, the sub jobs have already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) * completed so we can free everything.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (job->master_job == job) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) mutex_destroy(&job->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) mempool_free(job, &kc->job_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) fn(read_err, write_err, context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (atomic_dec_and_test(&kc->nr_jobs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) wake_up(&kc->destroyq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) static void complete_io(unsigned long error, void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) struct kcopyd_job *job = (struct kcopyd_job *) context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) struct dm_kcopyd_client *kc = job->kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) io_job_finish(kc->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) if (error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) if (op_is_write(job->rw))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) job->write_err |= error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) job->read_err = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) if (!test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) push(&kc->complete_jobs, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) wake(kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (op_is_write(job->rw))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) push(&kc->complete_jobs, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) job->rw = WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) push(&kc->io_jobs, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) wake(kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * Request io on as many buffer heads as we can currently get for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * a particular job.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) static int run_io_job(struct kcopyd_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) struct dm_io_request io_req = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) .bi_op = job->rw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) .bi_op_flags = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) .mem.type = DM_IO_PAGE_LIST,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) .mem.ptr.pl = job->pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) .mem.offset = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) .notify.fn = complete_io,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) .notify.context = job,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) .client = job->kc->io_client,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) * If we need to write sequentially and some reads or writes failed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) * no point in continuing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if (test_bit(DM_KCOPYD_WRITE_SEQ, &job->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) job->master_job->write_err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) job->write_err = job->master_job->write_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) io_job_start(job->kc->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) if (job->rw == READ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) r = dm_io(&io_req, 1, &job->source, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) r = dm_io(&io_req, job->num_dests, job->dests, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) static int run_pages_job(struct kcopyd_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) unsigned nr_pages = dm_div_up(job->dests[0].count, PAGE_SIZE >> 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) r = kcopyd_get_pages(job->kc, nr_pages, &job->pages, job->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) if (!r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) /* this job is ready for io */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) push(&job->kc->io_jobs, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) if (r == -ENOMEM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) /* can't complete now */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) * Run through a list for as long as possible. Returns the count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) * of successful jobs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) int (*fn) (struct kcopyd_job *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) struct kcopyd_job *job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) int r, count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) while ((job = pop(jobs, kc))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) r = fn(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) if (r < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) /* error this rogue job */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (op_is_write(job->rw))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) job->write_err = (unsigned long) -1L;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) job->read_err = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) push(&kc->complete_jobs, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) wake(kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) if (r > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * We couldn't service this job ATM, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * push this job back onto the list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) push_head(jobs, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) * kcopyd does this every time it's woken up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) static void do_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) struct dm_kcopyd_client *kc = container_of(work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) struct dm_kcopyd_client, kcopyd_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) * The order that these are called is *very* important.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) * complete jobs can free some pages for pages jobs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) * Pages jobs when successful will jump onto the io jobs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) * list. io jobs call wake when they complete and it all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) * starts again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) spin_lock_irqsave(&kc->job_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) list_splice_tail_init(&kc->callback_jobs, &kc->complete_jobs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) spin_unlock_irqrestore(&kc->job_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) process_jobs(&kc->complete_jobs, kc, run_complete_job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) process_jobs(&kc->pages_jobs, kc, run_pages_job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) process_jobs(&kc->io_jobs, kc, run_io_job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * If we are copying a small region we just dispatch a single job
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) * to do the copy, otherwise the io has to be split up into many
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) * jobs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) static void dispatch_job(struct kcopyd_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) struct dm_kcopyd_client *kc = job->kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) atomic_inc(&kc->nr_jobs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (unlikely(!job->source.count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) push(&kc->callback_jobs, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) else if (job->pages == &zero_page_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) push(&kc->io_jobs, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) push(&kc->pages_jobs, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) wake(kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) static void segment_complete(int read_err, unsigned long write_err,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) /* FIXME: tidy this function */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) sector_t progress = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) sector_t count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) struct kcopyd_job *sub_job = (struct kcopyd_job *) context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) struct kcopyd_job *job = sub_job->master_job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) struct dm_kcopyd_client *kc = job->kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) mutex_lock(&job->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) /* update the error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) if (read_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) job->read_err = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) if (write_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) job->write_err |= write_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) * Only dispatch more work if there hasn't been an error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) if ((!job->read_err && !job->write_err) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) /* get the next chunk of work */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) progress = job->progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) count = job->source.count - progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) if (count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) if (count > kc->sub_job_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) count = kc->sub_job_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) job->progress += count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) mutex_unlock(&job->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) if (count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) *sub_job = *job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) sub_job->write_offset = progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) sub_job->source.sector += progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) sub_job->source.count = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) for (i = 0; i < job->num_dests; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) sub_job->dests[i].sector += progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) sub_job->dests[i].count = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) sub_job->fn = segment_complete;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) sub_job->context = sub_job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) dispatch_job(sub_job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) } else if (atomic_dec_and_test(&job->sub_jobs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) * Queue the completion callback to the kcopyd thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) * Some callers assume that all the completions are called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * from a single thread and don't race with each other.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) * We must not call the callback directly here because this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) * code may not be executing in the thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) push(&kc->complete_jobs, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) wake(kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * Create some sub jobs to share the work between them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) static void split_job(struct kcopyd_job *master_job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) atomic_inc(&master_job->kc->nr_jobs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) atomic_set(&master_job->sub_jobs, SPLIT_COUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) for (i = 0; i < SPLIT_COUNT; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) master_job[i + 1].master_job = master_job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) segment_complete(0, 0u, &master_job[i + 1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) void dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) unsigned int num_dests, struct dm_io_region *dests,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) unsigned int flags, dm_kcopyd_notify_fn fn, void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) struct kcopyd_job *job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * Allocate an array of jobs consisting of one master job
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) * followed by SPLIT_COUNT sub jobs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) job = mempool_alloc(&kc->job_pool, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) mutex_init(&job->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) * set up for the read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) job->kc = kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) job->flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) job->read_err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) job->write_err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) job->num_dests = num_dests;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) memcpy(&job->dests, dests, sizeof(*dests) * num_dests);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) * If one of the destination is a host-managed zoned block device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) * we need to write sequentially. If one of the destination is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) * host-aware device, then leave it to the caller to choose what to do.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) if (!test_bit(DM_KCOPYD_WRITE_SEQ, &job->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) for (i = 0; i < job->num_dests; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) if (bdev_zoned_model(dests[i].bdev) == BLK_ZONED_HM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) set_bit(DM_KCOPYD_WRITE_SEQ, &job->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) * If we need to write sequentially, errors cannot be ignored.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) if (test_bit(DM_KCOPYD_WRITE_SEQ, &job->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) clear_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) if (from) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) job->source = *from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) job->pages = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) job->rw = READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) memset(&job->source, 0, sizeof job->source);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) job->source.count = job->dests[0].count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) job->pages = &zero_page_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) * Use WRITE ZEROES to optimize zeroing if all dests support it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) job->rw = REQ_OP_WRITE_ZEROES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) for (i = 0; i < job->num_dests; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) if (!bdev_write_zeroes_sectors(job->dests[i].bdev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) job->rw = WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) job->fn = fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) job->context = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) job->master_job = job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) job->write_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) if (job->source.count <= kc->sub_job_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) dispatch_job(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) job->progress = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) split_job(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) EXPORT_SYMBOL(dm_kcopyd_copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) void dm_kcopyd_zero(struct dm_kcopyd_client *kc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) unsigned num_dests, struct dm_io_region *dests,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) unsigned flags, dm_kcopyd_notify_fn fn, void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) dm_kcopyd_copy(kc, NULL, num_dests, dests, flags, fn, context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) EXPORT_SYMBOL(dm_kcopyd_zero);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) dm_kcopyd_notify_fn fn, void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) struct kcopyd_job *job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) job = mempool_alloc(&kc->job_pool, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) memset(job, 0, sizeof(struct kcopyd_job));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) job->kc = kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) job->fn = fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) job->context = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) job->master_job = job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) atomic_inc(&kc->nr_jobs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) return job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) EXPORT_SYMBOL(dm_kcopyd_prepare_callback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) void dm_kcopyd_do_callback(void *j, int read_err, unsigned long write_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) struct kcopyd_job *job = j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) struct dm_kcopyd_client *kc = job->kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) job->read_err = read_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) job->write_err = write_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) push(&kc->callback_jobs, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) wake(kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) EXPORT_SYMBOL(dm_kcopyd_do_callback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) * Cancels a kcopyd job, eg. someone might be deactivating a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) * mirror.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) #if 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) int kcopyd_cancel(struct kcopyd_job *job, int block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) /* FIXME: finish */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) #endif /* 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) /*-----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) * Client setup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) *---------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) struct dm_kcopyd_client *dm_kcopyd_client_create(struct dm_kcopyd_throttle *throttle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) unsigned reserve_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) struct dm_kcopyd_client *kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) kc = kzalloc(sizeof(*kc), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) if (!kc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) spin_lock_init(&kc->job_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) INIT_LIST_HEAD(&kc->callback_jobs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) INIT_LIST_HEAD(&kc->complete_jobs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) INIT_LIST_HEAD(&kc->io_jobs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) INIT_LIST_HEAD(&kc->pages_jobs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) kc->throttle = throttle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) r = mempool_init_slab_pool(&kc->job_pool, MIN_JOBS, _job_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) goto bad_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) INIT_WORK(&kc->kcopyd_work, do_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) kc->kcopyd_wq = alloc_workqueue("kcopyd", WQ_MEM_RECLAIM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) if (!kc->kcopyd_wq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) goto bad_workqueue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) kc->sub_job_size = dm_get_kcopyd_subjob_size();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) reserve_pages = DIV_ROUND_UP(kc->sub_job_size << SECTOR_SHIFT, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) kc->pages = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) kc->nr_reserved_pages = kc->nr_free_pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) r = client_reserve_pages(kc, reserve_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) goto bad_client_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) kc->io_client = dm_io_client_create();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) if (IS_ERR(kc->io_client)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) r = PTR_ERR(kc->io_client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) goto bad_io_client;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) init_waitqueue_head(&kc->destroyq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) atomic_set(&kc->nr_jobs, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) return kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) bad_io_client:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) client_free_pages(kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) bad_client_pages:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) destroy_workqueue(kc->kcopyd_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) bad_workqueue:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) mempool_exit(&kc->job_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) bad_slab:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) kfree(kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) return ERR_PTR(r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) EXPORT_SYMBOL(dm_kcopyd_client_create);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) /* Wait for completion of all jobs submitted by this client. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) BUG_ON(!list_empty(&kc->callback_jobs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) BUG_ON(!list_empty(&kc->complete_jobs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) BUG_ON(!list_empty(&kc->io_jobs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) BUG_ON(!list_empty(&kc->pages_jobs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) destroy_workqueue(kc->kcopyd_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) dm_io_client_destroy(kc->io_client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) client_free_pages(kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) mempool_exit(&kc->job_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) kfree(kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) EXPORT_SYMBOL(dm_kcopyd_client_destroy);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags