^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2007 Oracle. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 2014 Fujitsu. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/freezer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include "async-thread.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include "ctree.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) WORK_DONE_BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) WORK_ORDER_DONE_BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) WORK_HIGH_PRIO_BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define NO_THRESHOLD (-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define DFT_THRESHOLD (32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) struct __btrfs_workqueue {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) struct workqueue_struct *normal_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) /* File system this workqueue services */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) struct btrfs_fs_info *fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) /* List head pointing to ordered work list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) struct list_head ordered_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /* Spinlock for ordered_list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) spinlock_t list_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /* Thresholding related variants */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) atomic_t pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /* Up limit of concurrency workers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) int limit_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /* Current number of concurrency workers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) int current_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) /* Threshold to change current_active */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) int thresh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) unsigned int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) spinlock_t thres_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct btrfs_workqueue {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct __btrfs_workqueue *normal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct __btrfs_workqueue *high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct btrfs_fs_info * __pure btrfs_workqueue_owner(const struct __btrfs_workqueue *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) return wq->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct btrfs_fs_info * __pure btrfs_work_owner(const struct btrfs_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) return work->wq->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) bool btrfs_workqueue_normal_congested(const struct btrfs_workqueue *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * We could compare wq->normal->pending with num_online_cpus()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * to support "thresh == NO_THRESHOLD" case, but it requires
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * moving up atomic_inc/dec in thresh_queue/exec_hook. Let's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * postpone it until someone needs the support of that case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) if (wq->normal->thresh == NO_THRESHOLD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) return atomic_read(&wq->normal->pending) > wq->normal->thresh * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static struct __btrfs_workqueue *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) __btrfs_alloc_workqueue(struct btrfs_fs_info *fs_info, const char *name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) unsigned int flags, int limit_active, int thresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) struct __btrfs_workqueue *ret = kzalloc(sizeof(*ret), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) ret->fs_info = fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) ret->limit_active = limit_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) atomic_set(&ret->pending, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) if (thresh == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) thresh = DFT_THRESHOLD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) /* For low threshold, disabling threshold is a better choice */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (thresh < DFT_THRESHOLD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) ret->current_active = limit_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) ret->thresh = NO_THRESHOLD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * For threshold-able wq, let its concurrency grow on demand.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * Use minimal max_active at alloc time to reduce resource
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * usage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) ret->current_active = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) ret->thresh = thresh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) if (flags & WQ_HIGHPRI)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) ret->normal_wq = alloc_workqueue("btrfs-%s-high", flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) ret->current_active, name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) ret->normal_wq = alloc_workqueue("btrfs-%s", flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) ret->current_active, name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (!ret->normal_wq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) kfree(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) INIT_LIST_HEAD(&ret->ordered_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) spin_lock_init(&ret->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) spin_lock_init(&ret->thres_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) trace_btrfs_workqueue_alloc(ret, name, flags & WQ_HIGHPRI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) __btrfs_destroy_workqueue(struct __btrfs_workqueue *wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) struct btrfs_workqueue *btrfs_alloc_workqueue(struct btrfs_fs_info *fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) const char *name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) unsigned int flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) int limit_active,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) int thresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) struct btrfs_workqueue *ret = kzalloc(sizeof(*ret), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) ret->normal = __btrfs_alloc_workqueue(fs_info, name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) flags & ~WQ_HIGHPRI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) limit_active, thresh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) if (!ret->normal) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) kfree(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (flags & WQ_HIGHPRI) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) ret->high = __btrfs_alloc_workqueue(fs_info, name, flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) limit_active, thresh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if (!ret->high) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) __btrfs_destroy_workqueue(ret->normal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) kfree(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * Hook for threshold which will be called in btrfs_queue_work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * This hook WILL be called in IRQ handler context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * so workqueue_set_max_active MUST NOT be called in this hook
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) static inline void thresh_queue_hook(struct __btrfs_workqueue *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (wq->thresh == NO_THRESHOLD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) atomic_inc(&wq->pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * Hook for threshold which will be called before executing the work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * This hook is called in kthread content.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * So workqueue_set_max_active is called here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) static inline void thresh_exec_hook(struct __btrfs_workqueue *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) int new_current_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) long pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) int need_change = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) if (wq->thresh == NO_THRESHOLD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) atomic_dec(&wq->pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) spin_lock(&wq->thres_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * Use wq->count to limit the calling frequency of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * workqueue_set_max_active.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) wq->count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) wq->count %= (wq->thresh / 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) if (!wq->count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) new_current_active = wq->current_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * pending may be changed later, but it's OK since we really
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * don't need it so accurate to calculate new_max_active.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) pending = atomic_read(&wq->pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (pending > wq->thresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) new_current_active++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) if (pending < wq->thresh / 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) new_current_active--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) new_current_active = clamp_val(new_current_active, 1, wq->limit_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (new_current_active != wq->current_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) need_change = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) wq->current_active = new_current_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) spin_unlock(&wq->thres_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) if (need_change) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) workqueue_set_max_active(wq->normal_wq, wq->current_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) static void run_ordered_work(struct __btrfs_workqueue *wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) struct btrfs_work *self)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) struct list_head *list = &wq->ordered_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) struct btrfs_work *work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) spinlock_t *lock = &wq->list_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) bool free_self = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) spin_lock_irqsave(lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) if (list_empty(list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) work = list_entry(list->next, struct btrfs_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) ordered_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (!test_bit(WORK_DONE_BIT, &work->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * Orders all subsequent loads after reading WORK_DONE_BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * paired with the smp_mb__before_atomic in btrfs_work_helper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * this guarantees that the ordered function will see all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * updates from ordinary work function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * we are going to call the ordered done function, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * we leave the work item on the list as a barrier so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * that later work items that are done don't have their
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * functions called before this one returns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) if (test_and_set_bit(WORK_ORDER_DONE_BIT, &work->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) trace_btrfs_ordered_sched(work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) spin_unlock_irqrestore(lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) work->ordered_func(work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) /* now take the lock again and drop our item from the list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) spin_lock_irqsave(lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) list_del(&work->ordered_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) spin_unlock_irqrestore(lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (work == self) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * This is the work item that the worker is currently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * executing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * The kernel workqueue code guarantees non-reentrancy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * of work items. I.e., if a work item with the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * address and work function is queued twice, the second
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * execution is blocked until the first one finishes. A
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * work item may be freed and recycled with the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * work function; the workqueue code assumes that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * original work item cannot depend on the recycled work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * item in that case (see find_worker_executing_work()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * Note that different types of Btrfs work can depend on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * each other, and one type of work on one Btrfs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * filesystem may even depend on the same type of work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * on another Btrfs filesystem via, e.g., a loop device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * Therefore, we must not allow the current work item to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * be recycled until we are really done, otherwise we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * break the above assumption and can deadlock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) free_self = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * We don't want to call the ordered free functions with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * the lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) work->ordered_free(work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) /* NB: work must not be dereferenced past this point. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) trace_btrfs_all_work_done(wq->fs_info, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) spin_unlock_irqrestore(lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) if (free_self) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) self->ordered_free(self);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) /* NB: self must not be dereferenced past this point. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) trace_btrfs_all_work_done(wq->fs_info, self);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static void btrfs_work_helper(struct work_struct *normal_work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) struct btrfs_work *work = container_of(normal_work, struct btrfs_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) normal_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) struct __btrfs_workqueue *wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) int need_order = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) * We should not touch things inside work in the following cases:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * 1) after work->func() if it has no ordered_free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * Since the struct is freed in work->func().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * 2) after setting WORK_DONE_BIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * The work may be freed in other threads almost instantly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * So we save the needed things here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (work->ordered_func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) need_order = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) wq = work->wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) trace_btrfs_work_sched(work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) thresh_exec_hook(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) work->func(work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) if (need_order) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * Ensures all memory accesses done in the work function are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * ordered before setting the WORK_DONE_BIT. Ensuring the thread
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * which is going to executed the ordered work sees them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * Pairs with the smp_rmb in run_ordered_work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) set_bit(WORK_DONE_BIT, &work->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) run_ordered_work(wq, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) /* NB: work must not be dereferenced past this point. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) trace_btrfs_all_work_done(wq->fs_info, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) void btrfs_init_work(struct btrfs_work *work, btrfs_func_t func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) btrfs_func_t ordered_func, btrfs_func_t ordered_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) work->func = func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) work->ordered_func = ordered_func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) work->ordered_free = ordered_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) INIT_WORK(&work->normal_work, btrfs_work_helper);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) INIT_LIST_HEAD(&work->ordered_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) work->flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static inline void __btrfs_queue_work(struct __btrfs_workqueue *wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) struct btrfs_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) work->wq = wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) thresh_queue_hook(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) if (work->ordered_func) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) spin_lock_irqsave(&wq->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) list_add_tail(&work->ordered_list, &wq->ordered_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) spin_unlock_irqrestore(&wq->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) trace_btrfs_work_queued(work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) queue_work(wq->normal_wq, &work->normal_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) void btrfs_queue_work(struct btrfs_workqueue *wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) struct btrfs_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) struct __btrfs_workqueue *dest_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags) && wq->high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) dest_wq = wq->high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) dest_wq = wq->normal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) __btrfs_queue_work(dest_wq, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) __btrfs_destroy_workqueue(struct __btrfs_workqueue *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) destroy_workqueue(wq->normal_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) trace_btrfs_workqueue_destroy(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) kfree(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) void btrfs_destroy_workqueue(struct btrfs_workqueue *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (!wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (wq->high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) __btrfs_destroy_workqueue(wq->high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) __btrfs_destroy_workqueue(wq->normal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) kfree(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) void btrfs_workqueue_set_max(struct btrfs_workqueue *wq, int limit_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) if (!wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) wq->normal->limit_active = limit_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (wq->high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) wq->high->limit_active = limit_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) void btrfs_set_work_high_priority(struct btrfs_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) set_bit(WORK_HIGH_PRIO_BIT, &work->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) void btrfs_flush_workqueue(struct btrfs_workqueue *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (wq->high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) flush_workqueue(wq->high->normal_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) flush_workqueue(wq->normal->normal_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags