^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * raid5.c : Multiple Devices driver for Linux
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 1999, 2000 Ingo Molnar
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2002, 2003 H. Peter Anvin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * RAID-4/5/6 management functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Thanks to Penguin Computing for making the RAID-6 development possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * by donating a test server!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * BITMAP UNPLUGGING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * The sequencing for updating the bitmap reliably is a little
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * subtle (and I got it wrong the first time) so it deserves some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * explanation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * We group bitmap updates into batches. Each batch has a number.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * We may write out several batches at once, but that isn't very important.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * conf->seq_write is the number of the last batch successfully written.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * conf->seq_flush is the number of the last batch that was closed to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * new additions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * When we discover that we will need to write to any block in a stripe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * the number of the batch it will be in. This is seq_flush+1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * When we are ready to do a write, if that batch hasn't been written yet,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * we plug the array and queue the stripe for later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * When an unplug happens, we increment bm_flush, thus closing the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * batch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * When we notice that bm_flush > bm_write, we write out all pending updates
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * to the bitmap, and advance bm_write to where bm_flush was.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * This may occasionally write a bit out twice, but is sure never to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * miss any bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/raid/pq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/async_tx.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/async.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/ratelimit.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/nodemask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <trace/events/block.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/list_sort.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include "md.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include "raid5.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include "raid0.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include "md-bitmap.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include "raid5-log.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define UNSUPPORTED_MDDEV_FLAGS (1L << MD_FAILFAST_SUPPORTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define cpu_to_group(cpu) cpu_to_node(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define ANY_GROUP NUMA_NO_NODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static bool devices_handle_discard_safely = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) module_param(devices_handle_discard_safely, bool, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) MODULE_PARM_DESC(devices_handle_discard_safely,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static struct workqueue_struct *raid5_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return &conf->stripe_hashtbl[hash];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static inline int stripe_hash_locks_hash(struct r5conf *conf, sector_t sect)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) return (sect >> RAID5_STRIPE_SHIFT(conf)) & STRIPE_HASH_LOCKS_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) spin_lock_irq(conf->hash_locks + hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) spin_lock(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) spin_unlock(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) spin_unlock_irq(conf->hash_locks + hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) spin_lock_irq(conf->hash_locks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) spin_lock(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) spin_unlock(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) for (i = NR_STRIPE_HASH_LOCKS - 1; i; i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) spin_unlock(conf->hash_locks + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) spin_unlock_irq(conf->hash_locks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /* Find first data disk in a raid6 stripe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) static inline int raid6_d0(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (sh->ddf_layout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /* ddf always start from first device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) /* md starts just after Q block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) if (sh->qd_idx == sh->disks - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) return sh->qd_idx + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) static inline int raid6_next_disk(int disk, int raid_disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) disk++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) return (disk < raid_disks) ? disk : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /* When walking through the disks in a raid5, starting at raid6_d0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * We need to map each disk to a 'slot', where the data disks are slot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * is raid_disks-1. This help does that mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) int *count, int syndrome_disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) int slot = *count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) if (sh->ddf_layout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) (*count)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) if (idx == sh->pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) return syndrome_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) if (idx == sh->qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) return syndrome_disks + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) if (!sh->ddf_layout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) (*count)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) return slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) static void print_raid5_conf (struct r5conf *conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) static int stripe_operations_active(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) return sh->check_state || sh->reconstruct_state ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) test_bit(STRIPE_COMPUTE_RUN, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) static bool stripe_is_lowprio(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) !test_bit(STRIPE_R5C_CACHING, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) struct r5worker_group *group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) int thread_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) int i, cpu = sh->cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) if (!cpu_online(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) cpu = cpumask_any(cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) sh->cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (list_empty(&sh->lru)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) struct r5worker_group *group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) group = conf->worker_groups + cpu_to_group(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) if (stripe_is_lowprio(sh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) list_add_tail(&sh->lru, &group->loprio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) list_add_tail(&sh->lru, &group->handle_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) group->stripes_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) sh->group = group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (conf->worker_cnt_per_group == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) md_wakeup_thread(conf->mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) group = conf->worker_groups + cpu_to_group(sh->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) group->workers[0].working = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) /* at least one worker should run to avoid race */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /* wakeup more workers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (group->workers[i].working == false) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) group->workers[i].working = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) queue_work_on(sh->cpu, raid5_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) &group->workers[i].work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) thread_cnt--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct list_head *temp_inactive_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) int injournal = 0; /* number of date pages with R5_InJournal */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) BUG_ON(!list_empty(&sh->lru));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) BUG_ON(atomic_read(&conf->active_stripes)==0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (r5c_is_writeback(conf->log))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) for (i = sh->disks; i--; )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (test_bit(R5_InJournal, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) injournal++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * In the following cases, the stripe cannot be released to cached
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * lists. Therefore, we make the stripe write out and set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * STRIPE_HANDLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) * 1. when quiesce in r5c write back;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * 2. when resync is requested fot the stripe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) (conf->quiesce && r5c_is_writeback(conf->log) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (test_bit(STRIPE_R5C_CACHING, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) r5c_make_stripe_write_out(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (test_bit(STRIPE_HANDLE, &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) if (test_bit(STRIPE_DELAYED, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) list_add_tail(&sh->lru, &conf->delayed_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) sh->bm_seq - conf->seq_write > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) list_add_tail(&sh->lru, &conf->bitmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) clear_bit(STRIPE_DELAYED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) clear_bit(STRIPE_BIT_DELAY, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (conf->worker_cnt_per_group == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) if (stripe_is_lowprio(sh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) list_add_tail(&sh->lru,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) &conf->loprio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) list_add_tail(&sh->lru,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) &conf->handle_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) raid5_wakeup_stripe_thread(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) md_wakeup_thread(conf->mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) BUG_ON(stripe_operations_active(sh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) if (atomic_dec_return(&conf->preread_active_stripes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) < IO_THRESHOLD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) md_wakeup_thread(conf->mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) atomic_dec(&conf->active_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (!r5c_is_writeback(conf->log))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) list_add_tail(&sh->lru, temp_inactive_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (injournal == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) list_add_tail(&sh->lru, temp_inactive_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) else if (injournal == conf->raid_disks - conf->max_degraded) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) /* full stripe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) atomic_inc(&conf->r5c_cached_full_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) atomic_dec(&conf->r5c_cached_partial_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) r5c_check_cached_full_stripe(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * STRIPE_R5C_PARTIAL_STRIPE is set in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * r5c_try_caching_write(). No need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * set it again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) struct list_head *temp_inactive_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (atomic_dec_and_test(&sh->count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) do_release_stripe(conf, sh, temp_inactive_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) * Be careful: Only one task can add/delete stripes from temp_inactive_list at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * given time. Adding stripes only takes device lock, while deleting stripes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * only takes hash lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) static void release_inactive_stripe_list(struct r5conf *conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) struct list_head *temp_inactive_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) int hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) bool do_wakeup = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (hash == NR_STRIPE_HASH_LOCKS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) size = NR_STRIPE_HASH_LOCKS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) hash = NR_STRIPE_HASH_LOCKS - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) size = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) while (size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) struct list_head *list = &temp_inactive_list[size - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * We don't hold any lock here yet, raid5_get_active_stripe() might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * remove stripes from the list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) if (!list_empty_careful(list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) spin_lock_irqsave(conf->hash_locks + hash, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (list_empty(conf->inactive_list + hash) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) !list_empty(list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) atomic_dec(&conf->empty_inactive_list_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) list_splice_tail_init(list, conf->inactive_list + hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) do_wakeup = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) spin_unlock_irqrestore(conf->hash_locks + hash, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) size--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) hash--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) if (do_wakeup) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) wake_up(&conf->wait_for_stripe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) if (atomic_read(&conf->active_stripes) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) wake_up(&conf->wait_for_quiescent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) if (conf->retry_read_aligned)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) md_wakeup_thread(conf->mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) /* should hold conf->device_lock already */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) static int release_stripe_list(struct r5conf *conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) struct list_head *temp_inactive_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) struct stripe_head *sh, *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) int count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) struct llist_node *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) head = llist_del_all(&conf->released_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) head = llist_reverse_order(head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) llist_for_each_entry_safe(sh, t, head, release_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) int hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * Don't worry the bit is set here, because if the bit is set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * again, the count is always > 1. This is true for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * STRIPE_ON_UNPLUG_LIST bit too.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) hash = sh->hash_lock_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) __release_stripe(conf, sh, &temp_inactive_list[hash]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) void raid5_release_stripe(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) int hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) bool wakeup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) /* Avoid release_list until the last reference.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (atomic_add_unless(&sh->count, -1, 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) if (unlikely(!conf->mddev->thread) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) goto slow_path;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) wakeup = llist_add(&sh->release_list, &conf->released_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) if (wakeup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) md_wakeup_thread(conf->mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) slow_path:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) INIT_LIST_HEAD(&list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) hash = sh->hash_lock_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) do_release_stripe(conf, sh, &list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) spin_unlock_irqrestore(&conf->device_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) release_inactive_stripe_list(conf, &list, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) static inline void remove_hash(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) pr_debug("remove_hash(), stripe %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) hlist_del_init(&sh->hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) struct hlist_head *hp = stripe_hash(conf, sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) pr_debug("insert_hash(), stripe %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) hlist_add_head(&sh->hash, hp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) /* find an idle stripe, make sure it is unhashed, and return it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) struct stripe_head *sh = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) struct list_head *first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) if (list_empty(conf->inactive_list + hash))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) first = (conf->inactive_list + hash)->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) sh = list_entry(first, struct stripe_head, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) list_del_init(first);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) remove_hash(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) atomic_inc(&conf->active_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) BUG_ON(hash != sh->hash_lock_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) if (list_empty(conf->inactive_list + hash))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) atomic_inc(&conf->empty_inactive_list_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) return sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) static void free_stripe_pages(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) struct page *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) /* Have not allocate page pool */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) if (!sh->pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) for (i = 0; i < sh->nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) p = sh->pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) if (p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) put_page(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) sh->pages[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) struct page *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) for (i = 0; i < sh->nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) /* The page have allocated. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) if (sh->pages[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) p = alloc_page(gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) if (!p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) free_stripe_pages(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) sh->pages[i] = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) init_stripe_shared_pages(struct stripe_head *sh, struct r5conf *conf, int disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) int nr_pages, cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (sh->pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) /* Each of the sh->dev[i] need one conf->stripe_size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) cnt = PAGE_SIZE / conf->stripe_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) nr_pages = (disks + cnt - 1) / cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (!sh->pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) sh->nr_pages = nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) sh->stripes_per_page = cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) static void shrink_buffers(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) int num = sh->raid_conf->pool_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) #if PAGE_SIZE == DEFAULT_STRIPE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) for (i = 0; i < num ; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) struct page *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) p = sh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) sh->dev[i].page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) put_page(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) for (i = 0; i < num; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) sh->dev[i].page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) free_stripe_pages(sh); /* Free pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) int num = sh->raid_conf->pool_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) #if PAGE_SIZE == DEFAULT_STRIPE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) for (i = 0; i < num; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) if (!(page = alloc_page(gfp))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) sh->dev[i].page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) sh->dev[i].orig_page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) sh->dev[i].offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) if (alloc_stripe_pages(sh, gfp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) for (i = 0; i < num; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) sh->dev[i].page = raid5_get_dev_page(sh, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) sh->dev[i].orig_page = sh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) sh->dev[i].offset = raid5_get_page_offset(sh, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) struct stripe_head *sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) int i, seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) BUG_ON(atomic_read(&sh->count) != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) BUG_ON(stripe_operations_active(sh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) pr_debug("init_stripe called, stripe %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) (unsigned long long)sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) seq = read_seqcount_begin(&conf->gen_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) sh->generation = conf->generation - previous;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) sh->sector = sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) stripe_set_idx(sector, conf, previous, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) sh->state = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) for (i = sh->disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) if (dev->toread || dev->read || dev->towrite || dev->written ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) test_bit(R5_LOCKED, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) pr_err("sector=%llx i=%d %p %p %p %p %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) (unsigned long long)sh->sector, i, dev->toread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) dev->read, dev->towrite, dev->written,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) test_bit(R5_LOCKED, &dev->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) dev->flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) dev->sector = raid5_compute_blocknr(sh, i, previous);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) if (read_seqcount_retry(&conf->gen_lock, seq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) sh->overwrite_disks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) insert_hash(conf, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) sh->cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) set_bit(STRIPE_BATCH_READY, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) short generation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) if (sh->sector == sector && sh->generation == generation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) return sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * Need to check if array has failed when deciding whether to:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * - start an array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) * - remove non-faulty devices
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * - add a spare
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) * - allow a reshape
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) * This determination is simple when no reshape is happening.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * However if there is a reshape, we need to carefully check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) * both the before and after sections.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * This is because some failed devices may only affect one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) * of the two sections, and some non-in_sync devices may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * be insync in the section most affected by failed devices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) int raid5_calc_degraded(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) int degraded, degraded2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) degraded = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) for (i = 0; i < conf->previous_raid_disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) if (rdev && test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) rdev = rcu_dereference(conf->disks[i].replacement);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) if (!rdev || test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) degraded++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) else if (test_bit(In_sync, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) /* not in-sync or faulty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) * If the reshape increases the number of devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) * this is being recovered by the reshape, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) * this 'previous' section is not in_sync.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) * If the number of devices is being reduced however,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) * the device can only be part of the array if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) * we are reverting a reshape, so this section will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * be in-sync.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) if (conf->raid_disks >= conf->previous_raid_disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) degraded++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) if (conf->raid_disks == conf->previous_raid_disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) return degraded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) degraded2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) for (i = 0; i < conf->raid_disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) if (rdev && test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) rdev = rcu_dereference(conf->disks[i].replacement);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) if (!rdev || test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) degraded2++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) else if (test_bit(In_sync, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) /* not in-sync or faulty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * If reshape increases the number of devices, this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * section has already been recovered, else it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * almost certainly hasn't.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if (conf->raid_disks <= conf->previous_raid_disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) degraded2++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) if (degraded2 > degraded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return degraded2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) return degraded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) static int has_failed(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) int degraded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) if (conf->mddev->reshape_position == MaxSector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) return conf->mddev->degraded > conf->max_degraded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) degraded = raid5_calc_degraded(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) if (degraded > conf->max_degraded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) struct stripe_head *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) raid5_get_active_stripe(struct r5conf *conf, sector_t sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) int previous, int noblock, int noquiesce)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) int hash = stripe_hash_locks_hash(conf, sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) int inc_empty_inactive_list_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) spin_lock_irq(conf->hash_locks + hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) wait_event_lock_irq(conf->wait_for_quiescent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) conf->quiesce == 0 || noquiesce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) *(conf->hash_locks + hash));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) sh = __find_stripe(conf, sector, conf->generation - previous);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) if (!sh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) sh = get_free_stripe(conf, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) if (!sh && !test_bit(R5_DID_ALLOC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) &conf->cache_state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) set_bit(R5_ALLOC_MORE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) &conf->cache_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (noblock && sh == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) r5c_check_stripe_cache_usage(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) if (!sh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) set_bit(R5_INACTIVE_BLOCKED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) &conf->cache_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) r5l_wake_reclaim(conf->log, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) wait_event_lock_irq(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) conf->wait_for_stripe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) !list_empty(conf->inactive_list + hash) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) (atomic_read(&conf->active_stripes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) < (conf->max_nr_stripes * 3 / 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) || !test_bit(R5_INACTIVE_BLOCKED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) &conf->cache_state)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) *(conf->hash_locks + hash));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) clear_bit(R5_INACTIVE_BLOCKED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) &conf->cache_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) init_stripe(sh, sector, previous);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) } else if (!atomic_inc_not_zero(&sh->count)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) spin_lock(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) if (!atomic_read(&sh->count)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) if (!test_bit(STRIPE_HANDLE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) atomic_inc(&conf->active_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) BUG_ON(list_empty(&sh->lru) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) !test_bit(STRIPE_EXPANDING, &sh->state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) inc_empty_inactive_list_flag = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) if (!list_empty(conf->inactive_list + hash))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) inc_empty_inactive_list_flag = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) list_del_init(&sh->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) if (list_empty(conf->inactive_list + hash) && inc_empty_inactive_list_flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) atomic_inc(&conf->empty_inactive_list_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) if (sh->group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) sh->group->stripes_cnt--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) sh->group = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) spin_unlock(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) } while (sh == NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) spin_unlock_irq(conf->hash_locks + hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) return sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) static bool is_full_stripe_write(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) __acquires(&sh1->stripe_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) __acquires(&sh2->stripe_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) if (sh1 > sh2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) spin_lock_irq(&sh2->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) spin_lock_nested(&sh1->stripe_lock, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) spin_lock_irq(&sh1->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) spin_lock_nested(&sh2->stripe_lock, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) __releases(&sh1->stripe_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) __releases(&sh2->stripe_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) spin_unlock(&sh1->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) spin_unlock_irq(&sh2->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) /* Only freshly new full stripe normal write stripe can be added to a batch list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) static bool stripe_can_batch(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) if (raid5_has_log(conf) || raid5_has_ppl(conf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) return test_bit(STRIPE_BATCH_READY, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) !test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) is_full_stripe_write(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) /* we only do back search */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) struct stripe_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) sector_t head_sector, tmp_sec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) int hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) int dd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) int inc_empty_inactive_list_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) tmp_sec = sh->sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) if (!sector_div(tmp_sec, conf->chunk_sectors))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) hash = stripe_hash_locks_hash(conf, head_sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) spin_lock_irq(conf->hash_locks + hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) head = __find_stripe(conf, head_sector, conf->generation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) if (head && !atomic_inc_not_zero(&head->count)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) spin_lock(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if (!atomic_read(&head->count)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) if (!test_bit(STRIPE_HANDLE, &head->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) atomic_inc(&conf->active_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) BUG_ON(list_empty(&head->lru) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) !test_bit(STRIPE_EXPANDING, &head->state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) inc_empty_inactive_list_flag = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) if (!list_empty(conf->inactive_list + hash))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) inc_empty_inactive_list_flag = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) list_del_init(&head->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) if (list_empty(conf->inactive_list + hash) && inc_empty_inactive_list_flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) atomic_inc(&conf->empty_inactive_list_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) if (head->group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) head->group->stripes_cnt--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) head->group = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) atomic_inc(&head->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) spin_unlock(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) spin_unlock_irq(conf->hash_locks + hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) if (!head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) if (!stripe_can_batch(head))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) lock_two_stripes(head, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) /* clear_batch_ready clear the flag */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) if (!stripe_can_batch(head) || !stripe_can_batch(sh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) goto unlock_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) if (sh->batch_head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) goto unlock_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) dd_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) dd_idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) goto unlock_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) if (head->batch_head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) spin_lock(&head->batch_head->batch_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) /* This batch list is already running */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) if (!stripe_can_batch(head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) spin_unlock(&head->batch_head->batch_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) goto unlock_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) * We must assign batch_head of this stripe within the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) * batch_lock, otherwise clear_batch_ready of batch head
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * stripe could clear BATCH_READY bit of this stripe and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) * this stripe->batch_head doesn't get assigned, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) * could confuse clear_batch_ready for this stripe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) sh->batch_head = head->batch_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * at this point, head's BATCH_READY could be cleared, but we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * can still add the stripe to batch list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) list_add(&sh->batch_list, &head->batch_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) spin_unlock(&head->batch_head->batch_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) head->batch_head = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) sh->batch_head = head->batch_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) spin_lock(&head->batch_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) list_add_tail(&sh->batch_list, &head->batch_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) spin_unlock(&head->batch_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) if (atomic_dec_return(&conf->preread_active_stripes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) < IO_THRESHOLD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) md_wakeup_thread(conf->mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) int seq = sh->bm_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) sh->batch_head->bm_seq > seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) seq = sh->batch_head->bm_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) sh->batch_head->bm_seq = seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) unlock_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) unlock_two_stripes(head, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) raid5_release_stripe(head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) /* Determine if 'data_offset' or 'new_data_offset' should be used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) * in this stripe_head.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) sector_t progress = conf->reshape_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) /* Need a memory barrier to make sure we see the value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) * of conf->generation, or ->data_offset that was set before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) * reshape_progress was updated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) if (progress == MaxSector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) if (sh->generation == conf->generation - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) /* We are in a reshape, and this is a new-generation stripe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) * so use new_data_offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) static void dispatch_bio_list(struct bio_list *tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) while ((bio = bio_list_pop(tmp)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) static int cmp_stripe(void *priv, struct list_head *a, struct list_head *b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) const struct r5pending_data *da = list_entry(a,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) struct r5pending_data, sibling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) const struct r5pending_data *db = list_entry(b,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) struct r5pending_data, sibling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) if (da->sector > db->sector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) if (da->sector < db->sector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) static void dispatch_defer_bios(struct r5conf *conf, int target,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) struct bio_list *list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) struct r5pending_data *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) struct list_head *first, *next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) int cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) if (conf->pending_data_cnt == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) list_sort(NULL, &conf->pending_list, cmp_stripe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) first = conf->pending_list.next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) /* temporarily move the head */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) if (conf->next_pending_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) list_move_tail(&conf->pending_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) &conf->next_pending_data->sibling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) while (!list_empty(&conf->pending_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) data = list_first_entry(&conf->pending_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) struct r5pending_data, sibling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) if (&data->sibling == first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) first = data->sibling.next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) next = data->sibling.next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) bio_list_merge(list, &data->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) list_move(&data->sibling, &conf->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) if (cnt >= target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) conf->pending_data_cnt -= cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) BUG_ON(conf->pending_data_cnt < 0 || cnt < target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) if (next != &conf->pending_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) conf->next_pending_data = list_entry(next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) struct r5pending_data, sibling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) conf->next_pending_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) /* list isn't empty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) if (first != &conf->pending_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) list_move_tail(&conf->pending_list, first);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) static void flush_deferred_bios(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) struct bio_list tmp = BIO_EMPTY_LIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) if (conf->pending_data_cnt == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) spin_lock(&conf->pending_bios_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) dispatch_defer_bios(conf, conf->pending_data_cnt, &tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) BUG_ON(conf->pending_data_cnt != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) spin_unlock(&conf->pending_bios_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) dispatch_bio_list(&tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) static void defer_issue_bios(struct r5conf *conf, sector_t sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) struct bio_list *bios)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) struct bio_list tmp = BIO_EMPTY_LIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) struct r5pending_data *ent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) spin_lock(&conf->pending_bios_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) ent = list_first_entry(&conf->free_list, struct r5pending_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) sibling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) list_move_tail(&ent->sibling, &conf->pending_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) ent->sector = sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) bio_list_init(&ent->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) bio_list_merge(&ent->bios, bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) conf->pending_data_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) if (conf->pending_data_cnt >= PENDING_IO_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) dispatch_defer_bios(conf, PENDING_IO_ONE_FLUSH, &tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) spin_unlock(&conf->pending_bios_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) dispatch_bio_list(&tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) raid5_end_read_request(struct bio *bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) raid5_end_write_request(struct bio *bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) int i, disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) struct stripe_head *head_sh = sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) struct bio_list pending_bios = BIO_EMPTY_LIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) bool should_defer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) if (log_stripe(sh, s) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) should_defer = conf->batch_bio_dispatch && conf->group_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) int op, op_flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) int replace_only = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) struct bio *bi, *rbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) struct md_rdev *rdev, *rrdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) sh = head_sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) op = REQ_OP_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) op_flags = REQ_FUA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) if (test_bit(R5_Discard, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) op = REQ_OP_DISCARD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) op = REQ_OP_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) else if (test_and_clear_bit(R5_WantReplace,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) &sh->dev[i].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) op = REQ_OP_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) replace_only = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) op_flags |= REQ_SYNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) bi = &sh->dev[i].req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) rbi = &sh->dev[i].rreq; /* For writing to replacement */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) rrdev = rcu_dereference(conf->disks[i].replacement);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) rdev = rcu_dereference(conf->disks[i].rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) if (!rdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) rdev = rrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) rrdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) if (op_is_write(op)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) if (replace_only)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) rdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) if (rdev == rrdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) /* We raced and saw duplicates */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) rrdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) rdev = rrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) rrdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) if (rdev && test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) rdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) if (rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) atomic_inc(&rdev->nr_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) if (rrdev && test_bit(Faulty, &rrdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) rrdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) if (rrdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) atomic_inc(&rrdev->nr_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) /* We have already checked bad blocks for reads. Now
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) * need to check for writes. We never accept write errors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) * on the replacement, so we don't to check rrdev.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) while (op_is_write(op) && rdev &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) test_bit(WriteErrorSeen, &rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) sector_t first_bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) int bad_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) int bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) &first_bad, &bad_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) if (!bad)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) if (bad < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) set_bit(BlockedBadBlocks, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) if (!conf->mddev->external &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) conf->mddev->sb_flags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) /* It is very unlikely, but we might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) * still need to write out the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) * bad block log - better give it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) * a chance*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) md_check_recovery(conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) * Because md_wait_for_blocked_rdev
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) * will dec nr_pending, we must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) * increment it first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) atomic_inc(&rdev->nr_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) md_wait_for_blocked_rdev(rdev, conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) /* Acknowledged bad block - skip the write */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) rdev_dec_pending(rdev, conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) rdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) if (rdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) if (s->syncing || s->expanding || s->expanded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) || s->replacing)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) md_sync_acct(rdev->bdev, RAID5_STRIPE_SECTORS(conf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) set_bit(STRIPE_IO_STARTED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) bio_set_dev(bi, rdev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) bio_set_op_attrs(bi, op, op_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) bi->bi_end_io = op_is_write(op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) ? raid5_end_write_request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) : raid5_end_read_request;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) bi->bi_private = sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) pr_debug("%s: for %llu schedule op %d on disc %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) __func__, (unsigned long long)sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) bi->bi_opf, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) if (sh != head_sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) atomic_inc(&head_sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) if (use_new_offset(conf, sh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) bi->bi_iter.bi_sector = (sh->sector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) + rdev->new_data_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) bi->bi_iter.bi_sector = (sh->sector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) + rdev->data_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) bi->bi_opf |= REQ_NOMERGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) if (!op_is_write(op) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) test_bit(R5_InJournal, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) * issuing read for a page in journal, this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) * must be preparing for prexor in rmw; read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * the data into orig_page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) sh->dev[i].vec.bv_page = sh->dev[i].orig_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) sh->dev[i].vec.bv_page = sh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) bi->bi_vcnt = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) bi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) bi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) bi->bi_write_hint = sh->dev[i].write_hint;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) if (!rrdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) * If this is discard request, set bi_vcnt 0. We don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) * want to confuse SCSI because SCSI will replace payload
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) if (op == REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) bi->bi_vcnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) if (rrdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) if (conf->mddev->gendisk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) trace_block_bio_remap(bi->bi_disk->queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) bi, disk_devt(conf->mddev->gendisk),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) sh->dev[i].sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) if (should_defer && op_is_write(op))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) bio_list_add(&pending_bios, bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) submit_bio_noacct(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) if (rrdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) if (s->syncing || s->expanding || s->expanded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) || s->replacing)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) md_sync_acct(rrdev->bdev, RAID5_STRIPE_SECTORS(conf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) set_bit(STRIPE_IO_STARTED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) bio_set_dev(rbi, rrdev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) bio_set_op_attrs(rbi, op, op_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) BUG_ON(!op_is_write(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) rbi->bi_end_io = raid5_end_write_request;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) rbi->bi_private = sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) pr_debug("%s: for %llu schedule op %d on "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) "replacement disc %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) __func__, (unsigned long long)sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) rbi->bi_opf, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) if (sh != head_sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) atomic_inc(&head_sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) if (use_new_offset(conf, sh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) rbi->bi_iter.bi_sector = (sh->sector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) + rrdev->new_data_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) rbi->bi_iter.bi_sector = (sh->sector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) + rrdev->data_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) sh->dev[i].rvec.bv_page = sh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) rbi->bi_vcnt = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) rbi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) rbi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) rbi->bi_write_hint = sh->dev[i].write_hint;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) * If this is discard request, set bi_vcnt 0. We don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) * want to confuse SCSI because SCSI will replace payload
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) if (op == REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) rbi->bi_vcnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) if (conf->mddev->gendisk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) trace_block_bio_remap(rbi->bi_disk->queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) rbi, disk_devt(conf->mddev->gendisk),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) sh->dev[i].sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) if (should_defer && op_is_write(op))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) bio_list_add(&pending_bios, rbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) submit_bio_noacct(rbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) if (!rdev && !rrdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) if (op_is_write(op))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) set_bit(STRIPE_DEGRADED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) pr_debug("skip op %d on disc %d for sector %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) bi->bi_opf, i, (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) clear_bit(R5_LOCKED, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) if (!head_sh->batch_head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) sh = list_first_entry(&sh->batch_list, struct stripe_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) batch_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) if (sh != head_sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) if (should_defer && !bio_list_empty(&pending_bios))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) defer_issue_bios(conf, head_sh->sector, &pending_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) static struct dma_async_tx_descriptor *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) async_copy_data(int frombio, struct bio *bio, struct page **page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) unsigned int poff, sector_t sector, struct dma_async_tx_descriptor *tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) struct stripe_head *sh, int no_skipcopy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) struct bio_vec bvl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) struct bvec_iter iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) struct page *bio_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) int page_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) enum async_tx_flags flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) if (bio->bi_iter.bi_sector >= sector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) if (frombio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) flags |= ASYNC_TX_FENCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) bio_for_each_segment(bvl, bio, iter) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) int len = bvl.bv_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) int clen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) int b_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) if (page_offset < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) b_offset = -page_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) page_offset += b_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) len -= b_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) if (len > 0 && page_offset + len > RAID5_STRIPE_SIZE(conf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) clen = RAID5_STRIPE_SIZE(conf) - page_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) clen = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) if (clen > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) b_offset += bvl.bv_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) bio_page = bvl.bv_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) if (frombio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) if (conf->skip_copy &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) b_offset == 0 && page_offset == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) clen == RAID5_STRIPE_SIZE(conf) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) !no_skipcopy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) *page = bio_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) tx = async_memcpy(*page, bio_page, page_offset + poff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) b_offset, clen, &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) tx = async_memcpy(bio_page, *page, b_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) page_offset + poff, clen, &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) /* chain the operations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) submit.depend_tx = tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) if (clen < len) /* hit end of page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) page_offset += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) return tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) static void ops_complete_biofill(void *stripe_head_ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) struct stripe_head *sh = stripe_head_ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) pr_debug("%s: stripe %llu\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) /* clear completed biofills */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) for (i = sh->disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) /* acknowledge completion of a biofill operation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) /* and check if we need to reply to a read request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) * new R5_Wantfill requests are held off until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) * !STRIPE_BIOFILL_RUN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) struct bio *rbi, *rbi2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) BUG_ON(!dev->read);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) rbi = dev->read;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) dev->read = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) while (rbi && rbi->bi_iter.bi_sector <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) dev->sector + RAID5_STRIPE_SECTORS(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) rbi2 = r5_next_bio(conf, rbi, dev->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) bio_endio(rbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) rbi = rbi2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) static void ops_run_biofill(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) struct dma_async_tx_descriptor *tx = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) pr_debug("%s: stripe %llu\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) for (i = sh->disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) if (test_bit(R5_Wantfill, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) struct bio *rbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) spin_lock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) dev->read = rbi = dev->toread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) dev->toread = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) spin_unlock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) while (rbi && rbi->bi_iter.bi_sector <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) dev->sector + RAID5_STRIPE_SECTORS(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) tx = async_copy_data(0, rbi, &dev->page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) dev->offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) dev->sector, tx, sh, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) rbi = r5_next_bio(conf, rbi, dev->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) async_trigger_callback(&submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) static void mark_target_uptodate(struct stripe_head *sh, int target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) struct r5dev *tgt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) if (target < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) tgt = &sh->dev[target];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) set_bit(R5_UPTODATE, &tgt->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) clear_bit(R5_Wantcompute, &tgt->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) static void ops_complete_compute(void *stripe_head_ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) struct stripe_head *sh = stripe_head_ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) pr_debug("%s: stripe %llu\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) /* mark the computed target(s) as uptodate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) mark_target_uptodate(sh, sh->ops.target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) mark_target_uptodate(sh, sh->ops.target2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) if (sh->check_state == check_state_compute_run)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) sh->check_state = check_state_compute_result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) /* return a pointer to the address conversion region of the scribble buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) static struct page **to_addr_page(struct raid5_percpu *percpu, int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) return percpu->scribble + i * percpu->scribble_obj_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) /* return a pointer to the address conversion region of the scribble buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) static addr_conv_t *to_addr_conv(struct stripe_head *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) struct raid5_percpu *percpu, int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) return (void *) (to_addr_page(percpu, i) + sh->disks + 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) * Return a pointer to record offset address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) static unsigned int *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) static struct dma_async_tx_descriptor *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) int disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) struct page **xor_srcs = to_addr_page(percpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) unsigned int *off_srcs = to_addr_offs(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) int target = sh->ops.target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) struct r5dev *tgt = &sh->dev[target];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) struct page *xor_dest = tgt->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) unsigned int off_dest = tgt->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) int count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) struct dma_async_tx_descriptor *tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) pr_debug("%s: stripe %llu block: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) __func__, (unsigned long long)sh->sector, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) if (i != target) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) off_srcs[count] = sh->dev[i].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) xor_srcs[count++] = sh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) if (unlikely(count == 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) return tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) /* set_syndrome_sources - populate source buffers for gen_syndrome
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) * @srcs - (struct page *) array of size sh->disks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) * @offs - (unsigned int) array of offset for each page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) * @sh - stripe_head to parse
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) * Populates srcs in proper layout order for the stripe and returns the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) * 'count' of sources to be used in a call to async_gen_syndrome. The P
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) * destination buffer is recorded in srcs[count] and the Q destination
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) * is recorded in srcs[count+1]].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) static int set_syndrome_sources(struct page **srcs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) unsigned int *offs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) struct stripe_head *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) int srctype)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) int disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) int d0_idx = raid6_d0(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) for (i = 0; i < disks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) srcs[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) i = d0_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) if (i == sh->qd_idx || i == sh->pd_idx ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) (srctype == SYNDROME_SRC_ALL) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) (srctype == SYNDROME_SRC_WANT_DRAIN &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) (test_bit(R5_Wantdrain, &dev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) test_bit(R5_InJournal, &dev->flags))) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) (srctype == SYNDROME_SRC_WRITTEN &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) (dev->written ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) test_bit(R5_InJournal, &dev->flags)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) if (test_bit(R5_InJournal, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) srcs[slot] = sh->dev[i].orig_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) srcs[slot] = sh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) * For R5_InJournal, PAGE_SIZE must be 4KB and will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) * not shared page. In that case, dev[i].offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) * is 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) offs[slot] = sh->dev[i].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) i = raid6_next_disk(i, disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) } while (i != d0_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) return syndrome_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) static struct dma_async_tx_descriptor *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) int disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) struct page **blocks = to_addr_page(percpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) unsigned int *offs = to_addr_offs(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) int target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) int qd_idx = sh->qd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) struct dma_async_tx_descriptor *tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) struct r5dev *tgt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) struct page *dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) unsigned int dest_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) if (sh->ops.target < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) target = sh->ops.target2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) else if (sh->ops.target2 < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) target = sh->ops.target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) /* we should only have one valid target */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) BUG_ON(target < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) pr_debug("%s: stripe %llu block: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) __func__, (unsigned long long)sh->sector, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) tgt = &sh->dev[target];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) dest = tgt->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) dest_off = tgt->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) if (target == qd_idx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) blocks[count] = NULL; /* regenerating p is not necessary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) BUG_ON(blocks[count+1] != dest); /* q should already be set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) ops_complete_compute, sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) to_addr_conv(sh, percpu, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) tx = async_gen_syndrome(blocks, offs, count+2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) /* Compute any data- or p-drive using XOR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) for (i = disks; i-- ; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) if (i == target || i == qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) offs[count] = sh->dev[i].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) blocks[count++] = sh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) NULL, ops_complete_compute, sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) to_addr_conv(sh, percpu, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) tx = async_xor_offs(dest, dest_off, blocks, offs, count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) return tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) static struct dma_async_tx_descriptor *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) int i, count, disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) int syndrome_disks = sh->ddf_layout ? disks : disks-2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) int d0_idx = raid6_d0(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) int faila = -1, failb = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) int target = sh->ops.target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) int target2 = sh->ops.target2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) struct r5dev *tgt = &sh->dev[target];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) struct r5dev *tgt2 = &sh->dev[target2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) struct dma_async_tx_descriptor *tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) struct page **blocks = to_addr_page(percpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) unsigned int *offs = to_addr_offs(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) pr_debug("%s: stripe %llu block1: %d block2: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) __func__, (unsigned long long)sh->sector, target, target2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) BUG_ON(target < 0 || target2 < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) /* we need to open-code set_syndrome_sources to handle the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) * slot number conversion for 'faila' and 'failb'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) for (i = 0; i < disks ; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) offs[i] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) blocks[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) i = d0_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) offs[slot] = sh->dev[i].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) blocks[slot] = sh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) if (i == target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) faila = slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) if (i == target2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) failb = slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) i = raid6_next_disk(i, disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) } while (i != d0_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) BUG_ON(faila == failb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) if (failb < faila)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) swap(faila, failb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) __func__, (unsigned long long)sh->sector, faila, failb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) if (failb == syndrome_disks+1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) /* Q disk is one of the missing disks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) if (faila == syndrome_disks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) /* Missing P+Q, just recompute */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) ops_complete_compute, sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) to_addr_conv(sh, percpu, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) return async_gen_syndrome(blocks, offs, syndrome_disks+2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) RAID5_STRIPE_SIZE(sh->raid_conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) struct page *dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) unsigned int dest_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) int data_target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) int qd_idx = sh->qd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) /* Missing D+Q: recompute D from P, then recompute Q */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) if (target == qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) data_target = target2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) data_target = target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) for (i = disks; i-- ; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) if (i == data_target || i == qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) offs[count] = sh->dev[i].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) blocks[count++] = sh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) dest = sh->dev[data_target].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) dest_off = sh->dev[data_target].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) init_async_submit(&submit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) NULL, NULL, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) to_addr_conv(sh, percpu, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) tx = async_xor_offs(dest, dest_off, blocks, offs, count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) RAID5_STRIPE_SIZE(sh->raid_conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) init_async_submit(&submit, ASYNC_TX_FENCE, tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) ops_complete_compute, sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) to_addr_conv(sh, percpu, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) return async_gen_syndrome(blocks, offs, count+2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) RAID5_STRIPE_SIZE(sh->raid_conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) ops_complete_compute, sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) to_addr_conv(sh, percpu, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) if (failb == syndrome_disks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) /* We're missing D+P. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) return async_raid6_datap_recov(syndrome_disks+2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) RAID5_STRIPE_SIZE(sh->raid_conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) faila,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) blocks, offs, &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) /* We're missing D+D. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) return async_raid6_2data_recov(syndrome_disks+2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) RAID5_STRIPE_SIZE(sh->raid_conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) faila, failb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) blocks, offs, &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) static void ops_complete_prexor(void *stripe_head_ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) struct stripe_head *sh = stripe_head_ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) pr_debug("%s: stripe %llu\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) if (r5c_is_writeback(sh->raid_conf->log))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) * raid5-cache write back uses orig_page during prexor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) * After prexor, it is time to free orig_page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) r5c_release_extra_page(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) static struct dma_async_tx_descriptor *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) struct dma_async_tx_descriptor *tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) int disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) struct page **xor_srcs = to_addr_page(percpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) unsigned int *off_srcs = to_addr_offs(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) int count = 0, pd_idx = sh->pd_idx, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) /* existing parity data subtracted */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) pr_debug("%s: stripe %llu\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) /* Only process blocks that are known to be uptodate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) if (test_bit(R5_InJournal, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) * For this case, PAGE_SIZE must be equal to 4KB and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) * page offset is zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) off_srcs[count] = dev->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) xor_srcs[count++] = dev->orig_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) } else if (test_bit(R5_Wantdrain, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) off_srcs[count] = dev->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) xor_srcs[count++] = dev->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) return tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) static struct dma_async_tx_descriptor *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) struct dma_async_tx_descriptor *tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) struct page **blocks = to_addr_page(percpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) unsigned int *offs = to_addr_offs(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) pr_debug("%s: stripe %llu\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) tx = async_gen_syndrome(blocks, offs, count+2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) return tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) static struct dma_async_tx_descriptor *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) int disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) struct stripe_head *head_sh = sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) pr_debug("%s: stripe %llu\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) struct r5dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) struct bio *chosen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) sh = head_sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) struct bio *wbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) * clear R5_InJournal, so when rewriting a page in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) * journal, it is not skipped by r5l_log_stripe()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) clear_bit(R5_InJournal, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) spin_lock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) chosen = dev->towrite;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) dev->towrite = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) sh->overwrite_disks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) BUG_ON(dev->written);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) wbi = dev->written = chosen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) spin_unlock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) WARN_ON(dev->page != dev->orig_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) while (wbi && wbi->bi_iter.bi_sector <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) dev->sector + RAID5_STRIPE_SECTORS(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) if (wbi->bi_opf & REQ_FUA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) set_bit(R5_WantFUA, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) if (wbi->bi_opf & REQ_SYNC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) set_bit(R5_SyncIO, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) if (bio_op(wbi) == REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) set_bit(R5_Discard, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) tx = async_copy_data(1, wbi, &dev->page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) dev->offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) dev->sector, tx, sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) r5c_is_writeback(conf->log));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) if (dev->page != dev->orig_page &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) !r5c_is_writeback(conf->log)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) set_bit(R5_SkipCopy, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) clear_bit(R5_UPTODATE, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) clear_bit(R5_OVERWRITE, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) wbi = r5_next_bio(conf, wbi, dev->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) if (head_sh->batch_head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) sh = list_first_entry(&sh->batch_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) struct stripe_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) batch_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) if (sh == head_sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) return tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) static void ops_complete_reconstruct(void *stripe_head_ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) struct stripe_head *sh = stripe_head_ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) int disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) int pd_idx = sh->pd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) int qd_idx = sh->qd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) bool fua = false, sync = false, discard = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) pr_debug("%s: stripe %llu\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) discard |= test_bit(R5_Discard, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) if (dev->written || i == pd_idx || i == qd_idx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) if (!discard && !test_bit(R5_SkipCopy, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) set_bit(R5_UPTODATE, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) if (test_bit(STRIPE_EXPAND_READY, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) set_bit(R5_Expanded, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) if (fua)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) set_bit(R5_WantFUA, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) if (sync)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) set_bit(R5_SyncIO, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) if (sh->reconstruct_state == reconstruct_state_drain_run)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) sh->reconstruct_state = reconstruct_state_drain_result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) sh->reconstruct_state = reconstruct_state_prexor_drain_result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) BUG_ON(sh->reconstruct_state != reconstruct_state_run);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) sh->reconstruct_state = reconstruct_state_result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) struct dma_async_tx_descriptor *tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) int disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) struct page **xor_srcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) unsigned int *off_srcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) int count, pd_idx = sh->pd_idx, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) struct page *xor_dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) unsigned int off_dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) int prexor = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) int j = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) struct stripe_head *head_sh = sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) int last_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) pr_debug("%s: stripe %llu\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) for (i = 0; i < sh->disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) if (pd_idx == i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) if (!test_bit(R5_Discard, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) if (i >= sh->disks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) set_bit(R5_Discard, &sh->dev[pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) ops_complete_reconstruct(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) xor_srcs = to_addr_page(percpu, j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) off_srcs = to_addr_offs(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) /* check if prexor is active which means only process blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) * that are part of a read-modify-write (written)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) prexor = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) if (head_sh->dev[i].written ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) test_bit(R5_InJournal, &head_sh->dev[i].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) off_srcs[count] = dev->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) xor_srcs[count++] = dev->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) xor_dest = sh->dev[pd_idx].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) off_dest = sh->dev[pd_idx].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) if (i != pd_idx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) off_srcs[count] = dev->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) xor_srcs[count++] = dev->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) /* 1/ if we prexor'd then the dest is reused as a source
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) * 2/ if we did not prexor then we are redoing the parity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) * for the synchronous xor case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) last_stripe = !head_sh->batch_head ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) list_first_entry(&sh->batch_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) struct stripe_head, batch_list) == head_sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) if (last_stripe) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) flags = ASYNC_TX_ACK |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) atomic_inc(&head_sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) to_addr_conv(sh, percpu, j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) init_async_submit(&submit, flags, tx, NULL, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) to_addr_conv(sh, percpu, j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) if (unlikely(count == 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) if (!last_stripe) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) j++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) sh = list_first_entry(&sh->batch_list, struct stripe_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) batch_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) struct dma_async_tx_descriptor *tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) struct page **blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) unsigned int *offs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) int count, i, j = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) struct stripe_head *head_sh = sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) int last_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) int synflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) unsigned long txflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) for (i = 0; i < sh->disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) if (sh->pd_idx == i || sh->qd_idx == i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) if (!test_bit(R5_Discard, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) if (i >= sh->disks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) ops_complete_reconstruct(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) blocks = to_addr_page(percpu, j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) offs = to_addr_offs(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) synflags = SYNDROME_SRC_WRITTEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) synflags = SYNDROME_SRC_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) txflags = ASYNC_TX_ACK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) count = set_syndrome_sources(blocks, offs, sh, synflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) last_stripe = !head_sh->batch_head ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) list_first_entry(&sh->batch_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) struct stripe_head, batch_list) == head_sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) if (last_stripe) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) atomic_inc(&head_sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) init_async_submit(&submit, txflags, tx, ops_complete_reconstruct,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) head_sh, to_addr_conv(sh, percpu, j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) init_async_submit(&submit, 0, tx, NULL, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) to_addr_conv(sh, percpu, j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) tx = async_gen_syndrome(blocks, offs, count+2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) if (!last_stripe) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) j++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) sh = list_first_entry(&sh->batch_list, struct stripe_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) batch_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) static void ops_complete_check(void *stripe_head_ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) struct stripe_head *sh = stripe_head_ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) pr_debug("%s: stripe %llu\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) sh->check_state = check_state_check_result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) int disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) int pd_idx = sh->pd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) int qd_idx = sh->qd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) struct page *xor_dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) unsigned int off_dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) struct page **xor_srcs = to_addr_page(percpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) unsigned int *off_srcs = to_addr_offs(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) struct dma_async_tx_descriptor *tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) pr_debug("%s: stripe %llu\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) xor_dest = sh->dev[pd_idx].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) off_dest = sh->dev[pd_idx].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) off_srcs[count] = off_dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) xor_srcs[count++] = xor_dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) if (i == pd_idx || i == qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) off_srcs[count] = sh->dev[i].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) xor_srcs[count++] = sh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) init_async_submit(&submit, 0, NULL, NULL, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) to_addr_conv(sh, percpu, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) tx = async_xor_val_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) RAID5_STRIPE_SIZE(sh->raid_conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) &sh->ops.zero_sum_result, &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) tx = async_trigger_callback(&submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) struct page **srcs = to_addr_page(percpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) unsigned int *offs = to_addr_offs(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) pr_debug("%s: stripe %llu checkp: %d\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) (unsigned long long)sh->sector, checkp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) if (!checkp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) srcs[count] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) sh, to_addr_conv(sh, percpu, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) async_syndrome_val(srcs, offs, count+2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) RAID5_STRIPE_SIZE(sh->raid_conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) int overlap_clear = 0, i, disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) struct dma_async_tx_descriptor *tx = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) int level = conf->level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) struct raid5_percpu *percpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) unsigned long cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) cpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) percpu = per_cpu_ptr(conf->percpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) ops_run_biofill(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) overlap_clear++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) if (level < 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) tx = ops_run_compute5(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) if (sh->ops.target2 < 0 || sh->ops.target < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) tx = ops_run_compute6_1(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) tx = ops_run_compute6_2(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) /* terminate the chain if reconstruct is not set to be run */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) async_tx_ack(tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) if (test_bit(STRIPE_OP_PREXOR, &ops_request)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) if (level < 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) tx = ops_run_prexor5(sh, percpu, tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) tx = ops_run_prexor6(sh, percpu, tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) if (test_bit(STRIPE_OP_PARTIAL_PARITY, &ops_request))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) tx = ops_run_partial_parity(sh, percpu, tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) tx = ops_run_biodrain(sh, tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) overlap_clear++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) if (level < 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) ops_run_reconstruct5(sh, percpu, tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) ops_run_reconstruct6(sh, percpu, tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) if (sh->check_state == check_state_run)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) ops_run_check_p(sh, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) else if (sh->check_state == check_state_run_q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) ops_run_check_pq(sh, percpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) else if (sh->check_state == check_state_run_pq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) ops_run_check_pq(sh, percpu, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) if (overlap_clear && !sh->batch_head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) if (test_and_clear_bit(R5_Overlap, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) wake_up(&sh->raid_conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) kfree(sh->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) if (sh->ppl_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) __free_page(sh->ppl_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) kmem_cache_free(sc, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) int disks, struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) sh = kmem_cache_zalloc(sc, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) if (sh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) spin_lock_init(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) spin_lock_init(&sh->batch_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) INIT_LIST_HEAD(&sh->batch_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) INIT_LIST_HEAD(&sh->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) INIT_LIST_HEAD(&sh->r5c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) INIT_LIST_HEAD(&sh->log_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) atomic_set(&sh->count, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) sh->raid_conf = conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) sh->log_start = MaxSector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) for (i = 0; i < disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) bio_init(&dev->req, &dev->vec, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) bio_init(&dev->rreq, &dev->rvec, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) if (raid5_has_ppl(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) sh->ppl_page = alloc_page(gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) if (!sh->ppl_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) free_stripe(sc, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) if (init_stripe_shared_pages(sh, conf, disks)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) free_stripe(sc, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) return sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) if (!sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) if (grow_buffers(sh, gfp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) shrink_buffers(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) free_stripe(conf->slab_cache, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) sh->hash_lock_index =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) /* we just created an active stripe so... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) atomic_inc(&conf->active_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) conf->max_nr_stripes++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) static int grow_stripes(struct r5conf *conf, int num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) struct kmem_cache *sc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) size_t namelen = sizeof(conf->cache_name[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) int devs = max(conf->raid_disks, conf->previous_raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) if (conf->mddev->gendisk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) snprintf(conf->cache_name[0], namelen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) "raid%d-%s", conf->level, mdname(conf->mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) snprintf(conf->cache_name[0], namelen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) "raid%d-%p", conf->level, conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) snprintf(conf->cache_name[1], namelen, "%.27s-alt", conf->cache_name[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) conf->active_name = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) sc = kmem_cache_create(conf->cache_name[conf->active_name],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) 0, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) if (!sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) conf->slab_cache = sc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) conf->pool_size = devs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) while (num--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) if (!grow_one_stripe(conf, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) * scribble_alloc - allocate percpu scribble buffer for required size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) * of the scribble region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) * @percpu: from for_each_present_cpu() of the caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) * @num: total number of disks in the array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) * @cnt: scribble objs count for required size of the scribble region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) * The scribble buffer size must be enough to contain:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) * 1/ a struct page pointer for each device in the array +2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) * 2/ room to convert each entry in (1) to its corresponding dma
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) * (dma_map_page()) or page (page_address()) address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) * calculate over all devices (not just the data blocks), using zeros in place
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) * of the P and Q blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) static int scribble_alloc(struct raid5_percpu *percpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) int num, int cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) size_t obj_size =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) sizeof(struct page *) * (num + 2) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) sizeof(addr_conv_t) * (num + 2) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) sizeof(unsigned int) * (num + 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) void *scribble;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) * If here is in raid array suspend context, it is in memalloc noio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) * context as well, there is no potential recursive memory reclaim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) * I/Os with the GFP_KERNEL flag.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) if (!scribble)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) kvfree(percpu->scribble);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) percpu->scribble = scribble;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) percpu->scribble_obj_size = obj_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) unsigned long cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) * Never shrink. And mddev_suspend() could deadlock if this is called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) * from raid5d. In that case, scribble_disks and scribble_sectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) * should equal to new_disks and new_sectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) if (conf->scribble_disks >= new_disks &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) conf->scribble_sectors >= new_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) mddev_suspend(conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) get_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) for_each_present_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) struct raid5_percpu *percpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) percpu = per_cpu_ptr(conf->percpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) err = scribble_alloc(percpu, new_disks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) new_sectors / RAID5_STRIPE_SECTORS(conf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) put_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) mddev_resume(conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) conf->scribble_disks = new_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) conf->scribble_sectors = new_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) static int resize_stripes(struct r5conf *conf, int newsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) /* Make all the stripes able to hold 'newsize' devices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) * New slots in each stripe get 'page' set to a new page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) * This happens in stages:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) * 1/ create a new kmem_cache and allocate the required number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) * stripe_heads.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) * 2/ gather all the old stripe_heads and transfer the pages across
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) * to the new stripe_heads. This will have the side effect of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) * freezing the array as once all stripe_heads have been collected,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) * no IO will be possible. Old stripe heads are freed once their
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) * pages have been transferred over, and the old kmem_cache is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) * freed when all stripes are done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) * 3/ reallocate conf->disks to be suitable bigger. If this fails,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) * we simple return a failure status - no need to clean anything up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) * 4/ allocate new pages for the new slots in the new stripe_heads.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) * If this fails, we don't bother trying the shrink the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) * stripe_heads down again, we just leave them as they are.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) * As each stripe_head is processed the new one is released into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) * active service.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) * Once step2 is started, we cannot afford to wait for a write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) * so we use GFP_NOIO allocations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) struct stripe_head *osh, *nsh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) LIST_HEAD(newstripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) struct disk_info *ndisks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) struct kmem_cache *sc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) int hash, cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) md_allow_write(conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) /* Step 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) 0, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) if (!sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) /* Need to ensure auto-resizing doesn't interfere */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) mutex_lock(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) for (i = conf->max_nr_stripes; i; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) nsh = alloc_stripe(sc, GFP_KERNEL, newsize, conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) if (!nsh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) list_add(&nsh->lru, &newstripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) if (i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) /* didn't get enough, give up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) while (!list_empty(&newstripes)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) nsh = list_entry(newstripes.next, struct stripe_head, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) list_del(&nsh->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) free_stripe(sc, nsh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) kmem_cache_destroy(sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) mutex_unlock(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) /* Step 2 - Must use GFP_NOIO now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) * OK, we have enough stripes, start collecting inactive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) * stripes and copying them over
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) hash = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) list_for_each_entry(nsh, &newstripes, lru) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) lock_device_hash_lock(conf, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) wait_event_cmd(conf->wait_for_stripe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) !list_empty(conf->inactive_list + hash),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) unlock_device_hash_lock(conf, hash),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) lock_device_hash_lock(conf, hash));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) osh = get_free_stripe(conf, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) unlock_device_hash_lock(conf, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) for (i = 0; i < osh->nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) nsh->pages[i] = osh->pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) osh->pages[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) for(i=0; i<conf->pool_size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) nsh->dev[i].page = osh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) nsh->dev[i].orig_page = osh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) nsh->dev[i].offset = osh->dev[i].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) nsh->hash_lock_index = hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) free_stripe(conf->slab_cache, osh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) hash++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) kmem_cache_destroy(conf->slab_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) /* Step 3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) * At this point, we are holding all the stripes so the array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) * is completely stalled, so now is a good time to resize
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) * conf->disks and the scribble region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) ndisks = kcalloc(newsize, sizeof(struct disk_info), GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) if (ndisks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) for (i = 0; i < conf->pool_size; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) ndisks[i] = conf->disks[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) for (i = conf->pool_size; i < newsize; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) ndisks[i].extra_page = alloc_page(GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) if (!ndisks[i].extra_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) for (i = conf->pool_size; i < newsize; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) if (ndisks[i].extra_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) put_page(ndisks[i].extra_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) kfree(ndisks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) kfree(conf->disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) conf->disks = ndisks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) conf->slab_cache = sc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) conf->active_name = 1-conf->active_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) /* Step 4, return new stripes to service */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) while(!list_empty(&newstripes)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) nsh = list_entry(newstripes.next, struct stripe_head, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) list_del_init(&nsh->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) for (i = 0; i < nsh->nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) if (nsh->pages[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) nsh->pages[i] = alloc_page(GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) if (!nsh->pages[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) for (i = conf->raid_disks; i < newsize; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) if (nsh->dev[i].page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) nsh->dev[i].page = raid5_get_dev_page(nsh, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) nsh->dev[i].orig_page = nsh->dev[i].page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) nsh->dev[i].offset = raid5_get_page_offset(nsh, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) for (i=conf->raid_disks; i < newsize; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) if (nsh->dev[i].page == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) struct page *p = alloc_page(GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) nsh->dev[i].page = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) nsh->dev[i].orig_page = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) nsh->dev[i].offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) raid5_release_stripe(nsh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) /* critical section pass, GFP_NOIO no longer needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) conf->pool_size = newsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) mutex_unlock(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) static int drop_one_stripe(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) spin_lock_irq(conf->hash_locks + hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) sh = get_free_stripe(conf, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) spin_unlock_irq(conf->hash_locks + hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) if (!sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) BUG_ON(atomic_read(&sh->count));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) shrink_buffers(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) free_stripe(conf->slab_cache, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) atomic_dec(&conf->active_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) conf->max_nr_stripes--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) static void shrink_stripes(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) while (conf->max_nr_stripes &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) drop_one_stripe(conf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) kmem_cache_destroy(conf->slab_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) conf->slab_cache = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) static void raid5_end_read_request(struct bio * bi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) struct stripe_head *sh = bi->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) int disks = sh->disks, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) char b[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) struct md_rdev *rdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) sector_t s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) for (i=0 ; i<disks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) if (bi == &sh->dev[i].req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) pr_debug("end_read_request %llu/%d, count: %d, error %d.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) (unsigned long long)sh->sector, i, atomic_read(&sh->count),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) bi->bi_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) if (i == disks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) bio_reset(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) /* If replacement finished while this request was outstanding,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) * 'replacement' might be NULL already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) * In that case it moved down to 'rdev'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) * rdev is not removed until all requests are finished.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) rdev = conf->disks[i].replacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) if (!rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) rdev = conf->disks[i].rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) if (use_new_offset(conf, sh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) s = sh->sector + rdev->new_data_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) s = sh->sector + rdev->data_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) if (!bi->bi_status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) set_bit(R5_UPTODATE, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) /* Note that this cannot happen on a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) * replacement device. We just fail those on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) * any error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) pr_info_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) "md/raid:%s: read error corrected (%lu sectors at %llu on %s)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) mdname(conf->mddev), RAID5_STRIPE_SECTORS(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) (unsigned long long)s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) bdevname(rdev->bdev, b));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) atomic_add(RAID5_STRIPE_SECTORS(conf), &rdev->corrected_errors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) clear_bit(R5_ReadError, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) clear_bit(R5_ReWrite, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) if (test_bit(R5_InJournal, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) * end read for a page in journal, this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) * must be preparing for prexor in rmw
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) set_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) if (atomic_read(&rdev->read_errors))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) atomic_set(&rdev->read_errors, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) const char *bdn = bdevname(rdev->bdev, b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) int retry = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) int set_bad = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) clear_bit(R5_UPTODATE, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) if (!(bi->bi_status == BLK_STS_PROTECTION))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) atomic_inc(&rdev->read_errors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) pr_warn_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) "md/raid:%s: read error on replacement device (sector %llu on %s).\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) mdname(conf->mddev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) (unsigned long long)s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) bdn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) else if (conf->mddev->degraded >= conf->max_degraded) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) set_bad = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) pr_warn_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) "md/raid:%s: read error not correctable (sector %llu on %s).\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) mdname(conf->mddev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) (unsigned long long)s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) bdn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) /* Oh, no!!! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) set_bad = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) pr_warn_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) "md/raid:%s: read error NOT corrected!! (sector %llu on %s).\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) mdname(conf->mddev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) (unsigned long long)s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) bdn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) } else if (atomic_read(&rdev->read_errors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) > conf->max_nr_stripes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) if (!test_bit(Faulty, &rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) pr_warn("md/raid:%s: %d read_errors > %d stripes\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) mdname(conf->mddev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) atomic_read(&rdev->read_errors),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) conf->max_nr_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) pr_warn("md/raid:%s: Too many read errors, failing device %s.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) mdname(conf->mddev), bdn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) retry = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) if (set_bad && test_bit(In_sync, &rdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) retry = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) if (retry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) if (sh->qd_idx >= 0 && sh->pd_idx == i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) set_bit(R5_ReadError, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) set_bit(R5_ReadError, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) clear_bit(R5_ReadError, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) clear_bit(R5_ReWrite, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) if (!(set_bad
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) && test_bit(In_sync, &rdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) && rdev_set_badblocks(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) md_error(conf->mddev, rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) rdev_dec_pending(rdev, conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) bio_reset(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) clear_bit(R5_LOCKED, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) static void raid5_end_write_request(struct bio *bi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) struct stripe_head *sh = bi->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) int disks = sh->disks, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) sector_t first_bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) int bad_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) int replacement = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) for (i = 0 ; i < disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) if (bi == &sh->dev[i].req) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) rdev = conf->disks[i].rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) if (bi == &sh->dev[i].rreq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) rdev = conf->disks[i].replacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) if (rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) replacement = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) /* rdev was removed and 'replacement'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) * replaced it. rdev is not removed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) * until all requests are finished.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) rdev = conf->disks[i].rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) pr_debug("end_write_request %llu/%d, count %d, error: %d.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) (unsigned long long)sh->sector, i, atomic_read(&sh->count),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) bi->bi_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) if (i == disks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) bio_reset(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) if (replacement) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) if (bi->bi_status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) md_error(conf->mddev, rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) else if (is_badblock(rdev, sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) RAID5_STRIPE_SECTORS(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835) &first_bad, &bad_sectors))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) if (bi->bi_status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) set_bit(STRIPE_DEGRADED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) set_bit(WriteErrorSeen, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) set_bit(R5_WriteError, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) if (!test_and_set_bit(WantReplacement, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) set_bit(MD_RECOVERY_NEEDED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) &rdev->mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) } else if (is_badblock(rdev, sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) RAID5_STRIPE_SECTORS(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) &first_bad, &bad_sectors)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) set_bit(R5_MadeGood, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) if (test_bit(R5_ReadError, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) /* That was a successful write so make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) * sure it looks like we already did
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) * a re-write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) set_bit(R5_ReWrite, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) rdev_dec_pending(rdev, conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) if (sh->batch_head && bi->bi_status && !replacement)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) bio_reset(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) clear_bit(R5_LOCKED, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) if (sh->batch_head && sh != sh->batch_head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) raid5_release_stripe(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) static void raid5_error(struct mddev *mddev, struct md_rdev *rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) char b[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) pr_debug("raid456: error called\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) spin_lock_irqsave(&conf->device_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) if (test_bit(In_sync, &rdev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) mddev->degraded == conf->max_degraded) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) * Don't allow to achieve failed state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) * Don't try to recover this device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) conf->recovery_disabled = mddev->recovery_disabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) spin_unlock_irqrestore(&conf->device_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) set_bit(Faulty, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) clear_bit(In_sync, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) mddev->degraded = raid5_calc_degraded(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) spin_unlock_irqrestore(&conf->device_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) set_bit(MD_RECOVERY_INTR, &mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) set_bit(Blocked, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) set_mask_bits(&mddev->sb_flags, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) "md/raid:%s: Operation continuing on %d devices.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) mdname(mddev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) bdevname(rdev->bdev, b),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) mdname(mddev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) conf->raid_disks - mddev->degraded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) r5c_update_on_rdev_error(mddev, rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) * Input: a 'big' sector number,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) * Output: index of the data and parity disk, and the sector # in them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) int previous, int *dd_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) sector_t stripe, stripe2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) sector_t chunk_number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) unsigned int chunk_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) int pd_idx, qd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) int ddf_layout = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) sector_t new_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) int algorithm = previous ? conf->prev_algo
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) : conf->algorithm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) int sectors_per_chunk = previous ? conf->prev_chunk_sectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) : conf->chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) int raid_disks = previous ? conf->previous_raid_disks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) : conf->raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) int data_disks = raid_disks - conf->max_degraded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) /* First compute the information on this sector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) * Compute the chunk number and the sector offset inside the chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) chunk_offset = sector_div(r_sector, sectors_per_chunk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) chunk_number = r_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) * Compute the stripe number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) stripe = chunk_number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) *dd_idx = sector_div(stripe, data_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) stripe2 = stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) * Select the parity disk based on the user selected algorithm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) pd_idx = qd_idx = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) switch(conf->level) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) case 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) pd_idx = data_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) case 5:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) switch (algorithm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) case ALGORITHM_LEFT_ASYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) pd_idx = data_disks - sector_div(stripe2, raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) if (*dd_idx >= pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) (*dd_idx)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) case ALGORITHM_RIGHT_ASYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) pd_idx = sector_div(stripe2, raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) if (*dd_idx >= pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) (*dd_idx)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) case ALGORITHM_LEFT_SYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) pd_idx = data_disks - sector_div(stripe2, raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) case ALGORITHM_RIGHT_SYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) pd_idx = sector_div(stripe2, raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) case ALGORITHM_PARITY_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) pd_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) (*dd_idx)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) case ALGORITHM_PARITY_N:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) pd_idx = data_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) case 6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) switch (algorithm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) case ALGORITHM_LEFT_ASYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) qd_idx = pd_idx + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) if (pd_idx == raid_disks-1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) (*dd_idx)++; /* Q D D D P */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) qd_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) } else if (*dd_idx >= pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) (*dd_idx) += 2; /* D D P Q D */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) case ALGORITHM_RIGHT_ASYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) pd_idx = sector_div(stripe2, raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) qd_idx = pd_idx + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) if (pd_idx == raid_disks-1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) (*dd_idx)++; /* Q D D D P */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) qd_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) } else if (*dd_idx >= pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) (*dd_idx) += 2; /* D D P Q D */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) case ALGORITHM_LEFT_SYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) qd_idx = (pd_idx + 1) % raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) case ALGORITHM_RIGHT_SYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) pd_idx = sector_div(stripe2, raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) qd_idx = (pd_idx + 1) % raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) case ALGORITHM_PARITY_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) pd_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) qd_idx = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) (*dd_idx) += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) case ALGORITHM_PARITY_N:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) pd_idx = data_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) qd_idx = data_disks + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) case ALGORITHM_ROTATING_ZERO_RESTART:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) /* Exactly the same as RIGHT_ASYMMETRIC, but or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) * of blocks for computing Q is different.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) pd_idx = sector_div(stripe2, raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) qd_idx = pd_idx + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) if (pd_idx == raid_disks-1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) (*dd_idx)++; /* Q D D D P */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) qd_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) } else if (*dd_idx >= pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) (*dd_idx) += 2; /* D D P Q D */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) ddf_layout = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) case ALGORITHM_ROTATING_N_RESTART:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) /* Same a left_asymmetric, by first stripe is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) * D D D P Q rather than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) * Q D D D P
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) stripe2 += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) qd_idx = pd_idx + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) if (pd_idx == raid_disks-1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) (*dd_idx)++; /* Q D D D P */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) qd_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) } else if (*dd_idx >= pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) (*dd_idx) += 2; /* D D P Q D */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) ddf_layout = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) case ALGORITHM_ROTATING_N_CONTINUE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) /* Same as left_symmetric but Q is before P */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) ddf_layout = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) case ALGORITHM_LEFT_ASYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) /* RAID5 left_asymmetric, with Q on last device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) if (*dd_idx >= pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069) (*dd_idx)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) qd_idx = raid_disks - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) case ALGORITHM_RIGHT_ASYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074) pd_idx = sector_div(stripe2, raid_disks-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) if (*dd_idx >= pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) (*dd_idx)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) qd_idx = raid_disks - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) case ALGORITHM_LEFT_SYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) qd_idx = raid_disks - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) case ALGORITHM_RIGHT_SYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) pd_idx = sector_div(stripe2, raid_disks-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) qd_idx = raid_disks - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) case ALGORITHM_PARITY_0_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) pd_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) (*dd_idx)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) qd_idx = raid_disks - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) if (sh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) sh->pd_idx = pd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) sh->qd_idx = qd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) sh->ddf_layout = ddf_layout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) * Finally, compute the new sector number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) return new_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) int raid_disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) int data_disks = raid_disks - conf->max_degraded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) sector_t new_sector = sh->sector, check;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) int sectors_per_chunk = previous ? conf->prev_chunk_sectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) : conf->chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) int algorithm = previous ? conf->prev_algo
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) : conf->algorithm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) sector_t stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) int chunk_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) sector_t chunk_number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) int dummy1, dd_idx = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) sector_t r_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) struct stripe_head sh2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) chunk_offset = sector_div(new_sector, sectors_per_chunk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134) stripe = new_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) if (i == sh->pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) switch(conf->level) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) case 4: break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) case 5:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) switch (algorithm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) case ALGORITHM_LEFT_ASYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) case ALGORITHM_RIGHT_ASYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) if (i > sh->pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) i--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) case ALGORITHM_LEFT_SYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) case ALGORITHM_RIGHT_SYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149) if (i < sh->pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) i += raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) i -= (sh->pd_idx + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) case ALGORITHM_PARITY_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) i -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) case ALGORITHM_PARITY_N:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) case 6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) if (i == sh->qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) return 0; /* It is the Q disk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) switch (algorithm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) case ALGORITHM_LEFT_ASYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) case ALGORITHM_RIGHT_ASYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) case ALGORITHM_ROTATING_ZERO_RESTART:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) case ALGORITHM_ROTATING_N_RESTART:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) if (sh->pd_idx == raid_disks-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) i--; /* Q D D D P */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) else if (i > sh->pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173) i -= 2; /* D D P Q D */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) case ALGORITHM_LEFT_SYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) case ALGORITHM_RIGHT_SYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) if (sh->pd_idx == raid_disks-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) i--; /* Q D D D P */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) /* D D P Q D */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) if (i < sh->pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) i += raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) i -= (sh->pd_idx + 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) case ALGORITHM_PARITY_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187) i -= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189) case ALGORITHM_PARITY_N:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) case ALGORITHM_ROTATING_N_CONTINUE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) /* Like left_symmetric, but P is before Q */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) if (sh->pd_idx == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) i--; /* P D D D Q */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) /* D D Q P D */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) if (i < sh->pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) i += raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) i -= (sh->pd_idx + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) case ALGORITHM_LEFT_ASYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) case ALGORITHM_RIGHT_ASYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) if (i > sh->pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) i--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) case ALGORITHM_LEFT_SYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) case ALGORITHM_RIGHT_SYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) if (i < sh->pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) i += data_disks + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) i -= (sh->pd_idx + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) case ALGORITHM_PARITY_0_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) i -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) chunk_number = stripe * data_disks + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) r_sector = chunk_number * sectors_per_chunk + chunk_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) check = raid5_compute_sector(conf, r_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) previous, &dummy1, &sh2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) || sh2.qd_idx != sh->qd_idx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229) pr_warn("md/raid:%s: compute_blocknr: map not correct\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) mdname(conf->mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) return r_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) * There are cases where we want handle_stripe_dirtying() and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238) * schedule_reconstruction() to delay towrite to some dev of a stripe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) * This function checks whether we want to delay the towrite. Specifically,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) * we delay the towrite when:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) * 1. degraded stripe has a non-overwrite to the missing dev, AND this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) * stripe has data in journal (for other devices).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) * In this case, when reading data for the non-overwrite dev, it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) * necessary to handle complex rmw of write back cache (prexor with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) * orig_page, and xor with page). To keep read path simple, we would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) * like to flush data in journal to RAID disks first, so complex rmw
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) * is handled in the write patch (handle_stripe_dirtying).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) * 2. when journal space is critical (R5C_LOG_CRITICAL=1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) * It is important to be able to flush all stripes in raid5-cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) * Therefore, we need reserve some space on the journal device for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) * these flushes. If flush operation includes pending writes to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) * stripe, we need to reserve (conf->raid_disk + 1) pages per stripe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258) * for the flush out. If we exclude these pending writes from flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) * operation, we only need (conf->max_degraded + 1) pages per stripe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) * Therefore, excluding pending writes in these cases enables more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261) * efficient use of the journal device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) * Note: To make sure the stripe makes progress, we only delay
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) * towrite for stripes with data already in journal (injournal > 0).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) * When LOG_CRITICAL, stripes with injournal == 0 will be sent to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) * no_space_stripes list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) * 3. during journal failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) * In journal failure, we try to flush all cached data to raid disks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) * based on data in stripe cache. The array is read-only to upper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) * layers, so we would skip all pending writes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) static inline bool delay_towrite(struct r5conf *conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) struct r5dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) struct stripe_head_state *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) /* case 1 above */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) if (!test_bit(R5_OVERWRITE, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) !test_bit(R5_Insync, &dev->flags) && s->injournal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) /* case 2 above */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) s->injournal > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) /* case 3 above */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) if (s->log_failed && s->injournal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) int rcw, int expand)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) int level = conf->level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) if (rcw) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) * In some cases, handle_stripe_dirtying initially decided to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) * run rmw and allocates extra page for prexor. However, rcw is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) * cheaper later on. We need to free the extra page now,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) * because we won't be able to do that in ops_complete_prexor().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) r5c_release_extra_page(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) if (dev->towrite && !delay_towrite(conf, dev, s)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) set_bit(R5_Wantdrain, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315) if (!expand)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) clear_bit(R5_UPTODATE, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) } else if (test_bit(R5_InJournal, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) /* if we are not expanding this is a proper write request, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) * there will be bios with new data to be drained into the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) * stripe cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) if (!expand) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) if (!s->locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) /* False alarm, nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) sh->reconstruct_state = reconstruct_state_drain_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) sh->reconstruct_state = reconstruct_state_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) if (s->locked + conf->max_degraded == disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) atomic_inc(&conf->pending_full_writes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344) BUG_ON(level == 6 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) if (i == pd_idx || i == qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) if (dev->towrite &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) (test_bit(R5_UPTODATE, &dev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355) test_bit(R5_Wantcompute, &dev->flags))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) set_bit(R5_Wantdrain, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) clear_bit(R5_UPTODATE, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) } else if (test_bit(R5_InJournal, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) if (!s->locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) /* False alarm - nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) sh->reconstruct_state = reconstruct_state_prexor_drain_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) set_bit(STRIPE_OP_PREXOR, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371) set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374) /* keep the parity disk(s) locked while asynchronous operations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) * are in flight
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377) set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) if (level == 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) int qd_idx = sh->qd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) struct r5dev *dev = &sh->dev[qd_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) clear_bit(R5_UPTODATE, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390) if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) test_bit(STRIPE_OP_BIODRAIN, &s->ops_request) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) !test_bit(STRIPE_FULL_WRITE, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) test_bit(R5_Insync, &sh->dev[pd_idx].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) set_bit(STRIPE_OP_PARTIAL_PARITY, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) __func__, (unsigned long long)sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) s->locked, s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) * Each stripe/dev can have one or more bion attached.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) * toread/towrite point to the first in a chain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) * The bi_next chain must be in order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) int forwrite, int previous)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) struct bio **bip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) int firstwrite=0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413) pr_debug("adding bi b#%llu to stripe s#%llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) (unsigned long long)bi->bi_iter.bi_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) spin_lock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) sh->dev[dd_idx].write_hint = bi->bi_write_hint;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) /* Don't allow new IO added to stripes in batch list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) if (sh->batch_head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) goto overlap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) if (forwrite) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) bip = &sh->dev[dd_idx].towrite;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424) if (*bip == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) firstwrite = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) bip = &sh->dev[dd_idx].toread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429) if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) goto overlap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) bip = & (*bip)->bi_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) goto overlap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) if (forwrite && raid5_has_ppl(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) * With PPL only writes to consecutive data chunks within a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) * stripe are allowed because for a single stripe_head we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) * only have one PPL entry at a time, which describes one data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) * range. Not really an overlap, but wait_for_overlap can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) * used to handle this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444) sector_t sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) sector_t first = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) sector_t last = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) int count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) for (i = 0; i < sh->disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) if (i != sh->pd_idx &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) (i == dd_idx || sh->dev[i].towrite)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) sector = sh->dev[i].sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454) if (count == 0 || sector < first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) first = sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) if (sector > last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) last = sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) if (first + conf->chunk_sectors * (count - 1) != last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463) goto overlap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) if (!forwrite || previous)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) clear_bit(STRIPE_BATCH_READY, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469) BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) if (*bip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471) bi->bi_next = *bip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472) *bip = bi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) bio_inc_remaining(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) md_write_inc(conf->mddev, bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476) if (forwrite) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) /* check if page is covered */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) sector_t sector = sh->dev[dd_idx].sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) for (bi=sh->dev[dd_idx].towrite;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) bi && bi->bi_iter.bi_sector <= sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) if (bio_end_sector(bi) >= sector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) sector = bio_end_sector(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486) if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488) sh->overwrite_disks++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492) (unsigned long long)(*bip)->bi_iter.bi_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) (unsigned long long)sh->sector, dd_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495) if (conf->mddev->bitmap && firstwrite) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) /* Cannot hold spinlock over bitmap_startwrite,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) * but must ensure this isn't added to a batch until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) * we have added to the bitmap and set bm_seq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499) * So set STRIPE_BITMAP_PENDING to prevent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) * batching.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501) * If multiple add_stripe_bio() calls race here they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) * much all set STRIPE_BITMAP_PENDING. So only the first one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503) * to complete "bitmap_startwrite" gets to set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) * STRIPE_BIT_DELAY. This is important as once a stripe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) * is added to a batch, STRIPE_BIT_DELAY cannot be changed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) * any more.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) set_bit(STRIPE_BITMAP_PENDING, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) spin_unlock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510) md_bitmap_startwrite(conf->mddev->bitmap, sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) RAID5_STRIPE_SECTORS(conf), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) spin_lock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513) clear_bit(STRIPE_BITMAP_PENDING, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) if (!sh->batch_head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515) sh->bm_seq = conf->seq_flush+1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) set_bit(STRIPE_BIT_DELAY, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) spin_unlock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521) if (stripe_can_batch(sh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) stripe_add_to_batch_list(conf, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525) overlap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526) set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) spin_unlock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531) static void end_reshape(struct r5conf *conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534) struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) int sectors_per_chunk =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537) previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538) int dd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) int chunk_offset = sector_div(stripe, sectors_per_chunk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540) int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) raid5_compute_sector(conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) stripe * (disks - conf->max_degraded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544) *sectors_per_chunk + chunk_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545) previous,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) &dd_idx, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550) handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551) struct stripe_head_state *s, int disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556) struct bio *bi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557) int bitmap_end = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562) rdev = rcu_dereference(conf->disks[i].rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) if (rdev && test_bit(In_sync, &rdev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564) !test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565) atomic_inc(&rdev->nr_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567) rdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569) if (rdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570) if (!rdev_set_badblocks(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571) rdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572) sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) RAID5_STRIPE_SECTORS(conf), 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574) md_error(conf->mddev, rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) rdev_dec_pending(rdev, conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) spin_lock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579) /* fail all writes first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) bi = sh->dev[i].towrite;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581) sh->dev[i].towrite = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582) sh->overwrite_disks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) spin_unlock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584) if (bi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) bitmap_end = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587) log_stripe_write_finished(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) wake_up(&conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) while (bi && bi->bi_iter.bi_sector <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593) sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594) struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596) md_write_end(conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) bio_io_error(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598) bi = nextbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600) if (bitmap_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602) RAID5_STRIPE_SECTORS(conf), 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) bitmap_end = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) /* and fail all 'written' */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) bi = sh->dev[i].written;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606) sh->dev[i].written = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608) WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609) sh->dev[i].page = sh->dev[i].orig_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612) if (bi) bitmap_end = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613) while (bi && bi->bi_iter.bi_sector <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617) md_write_end(conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618) bio_io_error(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619) bi = bi2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622) /* fail any reads if this device is non-operational and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623) * the data has not reached the cache yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626) s->failed > conf->max_degraded &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627) (!test_bit(R5_Insync, &sh->dev[i].flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) test_bit(R5_ReadError, &sh->dev[i].flags))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629) spin_lock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) bi = sh->dev[i].toread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) sh->dev[i].toread = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632) spin_unlock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633) if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634) wake_up(&conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635) if (bi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) s->to_read--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637) while (bi && bi->bi_iter.bi_sector <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) struct bio *nextbi =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640) r5_next_bio(conf, bi, sh->dev[i].sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) bio_io_error(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643) bi = nextbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) if (bitmap_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647) md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) RAID5_STRIPE_SECTORS(conf), 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) /* If we were in the middle of a write the parity block might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) * still be locked - so just clear all R5_LOCKED flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652) clear_bit(R5_LOCKED, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654) s->to_write = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655) s->written = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657) if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) if (atomic_dec_and_test(&conf->pending_full_writes))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659) md_wakeup_thread(conf->mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) struct stripe_head_state *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666) int abort = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) clear_bit(STRIPE_SYNCING, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672) wake_up(&conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673) s->syncing = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) s->replacing = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675) /* There is nothing more to do for sync/check/repair.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) * Don't even need to abort as that is handled elsewhere
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) * if needed, and not always wanted e.g. if there is a known
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678) * bad block here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679) * For recover/replace we need to record a bad block on all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680) * non-sync devices, or abort the recovery
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) /* During recovery devices cannot be removed, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) * locking and refcounting of rdevs is not needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687) for (i = 0; i < conf->raid_disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689) if (rdev
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690) && !test_bit(Faulty, &rdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) && !test_bit(In_sync, &rdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692) && !rdev_set_badblocks(rdev, sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693) RAID5_STRIPE_SECTORS(conf), 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) abort = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) rdev = rcu_dereference(conf->disks[i].replacement);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696) if (rdev
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697) && !test_bit(Faulty, &rdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) && !test_bit(In_sync, &rdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699) && !rdev_set_badblocks(rdev, sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700) RAID5_STRIPE_SECTORS(conf), 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) abort = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704) if (abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) conf->recovery_disabled =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706) conf->mddev->recovery_disabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), !abort);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) static int want_replace(struct stripe_head *sh, int disk_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714) int rv = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717) rdev = rcu_dereference(sh->raid_conf->disks[disk_idx].replacement);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) if (rdev
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719) && !test_bit(Faulty, &rdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720) && !test_bit(In_sync, &rdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) && (rdev->recovery_offset <= sh->sector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722) || rdev->mddev->recovery_cp <= sh->sector))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) rv = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725) return rv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728) static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) int disk_idx, int disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731) struct r5dev *dev = &sh->dev[disk_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733) &sh->dev[s->failed_num[1]] };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735) bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) if (test_bit(R5_LOCKED, &dev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739) test_bit(R5_UPTODATE, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740) /* No point reading this as we already have it or have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741) * decided to get it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745) if (dev->toread ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746) (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747) /* We need this block to directly satisfy a request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750) if (s->syncing || s->expanding ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751) (s->replacing && want_replace(sh, disk_idx)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752) /* When syncing, or expanding we read everything.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753) * When replacing, we need the replaced block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757) if ((s->failed >= 1 && fdev[0]->toread) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) (s->failed >= 2 && fdev[1]->toread))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759) /* If we want to read from a failed device, then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760) * we need to actually read every other device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764) /* Sometimes neither read-modify-write nor reconstruct-write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765) * cycles can work. In those cases we read every block we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766) * can. Then the parity-update is certain to have enough to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) * work with.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768) * This can only be a problem when we need to write something,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769) * and some device has failed. If either of those tests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) * fail we need look no further.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) if (!s->failed || !s->to_write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775) if (test_bit(R5_Insync, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776) !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777) /* Pre-reads at not permitted until after short delay
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778) * to gather multiple requests. However if this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) * device is no Insync, the block could only be computed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780) * and there is no need to delay that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784) for (i = 0; i < s->failed && i < 2; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785) if (fdev[i]->towrite &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786) !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787) !test_bit(R5_OVERWRITE, &fdev[i]->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788) /* If we have a partial write to a failed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) * device, then we will need to reconstruct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790) * the content of that device, so all other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) * devices must be read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795) if (s->failed >= 2 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796) (fdev[i]->towrite ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797) s->failed_num[i] == sh->pd_idx ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798) s->failed_num[i] == sh->qd_idx) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799) !test_bit(R5_UPTODATE, &fdev[i]->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800) /* In max degraded raid6, If the failed disk is P, Q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801) * or we want to read the failed disk, we need to do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) * reconstruct-write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) force_rcw = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807) /* If we are forced to do a reconstruct-write, because parity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808) * cannot be trusted and we are currently recovering it, there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809) * is extra need to be careful.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) * If one of the devices that we would need to read, because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811) * it is not being overwritten (and maybe not written at all)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812) * is missing/faulty, then we need to read everything we can.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814) if (!force_rcw &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) sh->sector < sh->raid_conf->mddev->recovery_cp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816) /* reconstruct-write isn't being forced */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818) for (i = 0; i < s->failed && i < 2; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) if (s->failed_num[i] != sh->pd_idx &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820) s->failed_num[i] != sh->qd_idx &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821) !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) !test_bit(R5_OVERWRITE, &fdev[i]->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829) /* fetch_block - checks the given member device to see if its data needs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830) * to be read or computed to satisfy a request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832) * Returns 1 when no more member devices need to be checked, otherwise returns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833) * 0 to tell the loop in handle_stripe_fill to continue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835) static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836) int disk_idx, int disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838) struct r5dev *dev = &sh->dev[disk_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840) /* is the data in this block needed, and can we get it? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841) if (need_this_block(sh, s, disk_idx, disks)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842) /* we would like to get this block, possibly by computing it,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843) * otherwise read it if the backing disk is insync
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846) BUG_ON(test_bit(R5_Wantread, &dev->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850) * In the raid6 case if the only non-uptodate disk is P
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851) * then we already trusted P to compute the other failed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852) * drives. It is safe to compute rather than re-read P.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853) * In other cases we only compute blocks from failed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854) * devices, otherwise check/repair might fail to detect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) * a real inconsistency.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) if ((s->uptodate == disks - 1) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859) ((sh->qd_idx >= 0 && sh->pd_idx == disk_idx) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860) (s->failed && (disk_idx == s->failed_num[0] ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861) disk_idx == s->failed_num[1])))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862) /* have disk failed, and we're requested to fetch it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863) * do compute it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865) pr_debug("Computing stripe %llu block %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866) (unsigned long long)sh->sector, disk_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867) set_bit(STRIPE_COMPUTE_RUN, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869) set_bit(R5_Wantcompute, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) sh->ops.target = disk_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871) sh->ops.target2 = -1; /* no 2nd target */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872) s->req_compute = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873) /* Careful: from this point on 'uptodate' is in the eye
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) * of raid_run_ops which services 'compute' operations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) * before writes. R5_Wantcompute flags a block that will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876) * be R5_UPTODATE by the time it is needed for a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877) * subsequent operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879) s->uptodate++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881) } else if (s->uptodate == disks-2 && s->failed >= 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882) /* Computing 2-failure is *very* expensive; only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883) * do it if failed >= 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885) int other;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) for (other = disks; other--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887) if (other == disk_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889) if (!test_bit(R5_UPTODATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890) &sh->dev[other].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893) BUG_ON(other < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894) pr_debug("Computing stripe %llu blocks %d,%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895) (unsigned long long)sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896) disk_idx, other);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897) set_bit(STRIPE_COMPUTE_RUN, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898) set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899) set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900) set_bit(R5_Wantcompute, &sh->dev[other].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901) sh->ops.target = disk_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902) sh->ops.target2 = other;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903) s->uptodate += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904) s->req_compute = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906) } else if (test_bit(R5_Insync, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908) set_bit(R5_Wantread, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910) pr_debug("Reading block %d (sync=%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911) disk_idx, s->syncing);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919) * handle_stripe_fill - read or compute data to satisfy pending requests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921) static void handle_stripe_fill(struct stripe_head *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) struct stripe_head_state *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923) int disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927) /* look for blocks to read/compute, skip this if a compute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) * is already in flight, or if the stripe contents are in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929) * midst of changing due to a write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931) if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932) !sh->reconstruct_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935) * For degraded stripe with data in journal, do not handle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936) * read requests yet, instead, flush the stripe to raid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937) * disks first, this avoids handling complex rmw of write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) * back cache (prexor with orig_page, and then xor with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) * page) in the read path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941) if (s->injournal && s->failed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) if (test_bit(STRIPE_R5C_CACHING, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943) r5c_make_stripe_write_out(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947) for (i = disks; i--; )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948) if (fetch_block(sh, s, i, disks))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955) static void break_stripe_batch_list(struct stripe_head *head_sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956) unsigned long handle_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957) /* handle_stripe_clean_event
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958) * any written block on an uptodate or failed drive can be returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959) * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960) * never LOCKED, so we don't need to test 'failed' directly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962) static void handle_stripe_clean_event(struct r5conf *conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963) struct stripe_head *sh, int disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966) struct r5dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) int discard_pending = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968) struct stripe_head *head_sh = sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969) bool do_endio = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) for (i = disks; i--; )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972) if (sh->dev[i].written) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974) if (!test_bit(R5_LOCKED, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975) (test_bit(R5_UPTODATE, &dev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) test_bit(R5_Discard, &dev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977) test_bit(R5_SkipCopy, &dev->flags))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) /* We can return any write requests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979) struct bio *wbi, *wbi2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980) pr_debug("Return write for disc %d\n", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981) if (test_and_clear_bit(R5_Discard, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982) clear_bit(R5_UPTODATE, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984) WARN_ON(test_bit(R5_UPTODATE, &dev->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986) do_endio = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988) returnbi:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989) dev->page = dev->orig_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990) wbi = dev->written;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991) dev->written = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992) while (wbi && wbi->bi_iter.bi_sector <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993) dev->sector + RAID5_STRIPE_SECTORS(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) wbi2 = r5_next_bio(conf, wbi, dev->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995) md_write_end(conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996) bio_endio(wbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997) wbi = wbi2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999) md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000) RAID5_STRIPE_SECTORS(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001) !test_bit(STRIPE_DEGRADED, &sh->state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003) if (head_sh->batch_head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004) sh = list_first_entry(&sh->batch_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005) struct stripe_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006) batch_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) if (sh != head_sh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008) dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009) goto returnbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012) sh = head_sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013) dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014) } else if (test_bit(R5_Discard, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015) discard_pending = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018) log_stripe_write_finished(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020) if (!discard_pending &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021) test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022) int hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023) clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024) clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025) if (sh->qd_idx >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026) clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027) clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029) /* now that discard is done we can proceed with any sync */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030) clear_bit(STRIPE_DISCARD, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032) * SCSI discard will change some bio fields and the stripe has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) * no updated data, so remove it from hash list and the stripe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034) * will be reinitialized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036) unhash:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037) hash = sh->hash_lock_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038) spin_lock_irq(conf->hash_locks + hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039) remove_hash(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040) spin_unlock_irq(conf->hash_locks + hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041) if (head_sh->batch_head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) sh = list_first_entry(&sh->batch_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043) struct stripe_head, batch_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044) if (sh != head_sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045) goto unhash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047) sh = head_sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049) if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054) if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055) if (atomic_dec_and_test(&conf->pending_full_writes))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) md_wakeup_thread(conf->mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058) if (head_sh->batch_head && do_endio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059) break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063) * For RMW in write back cache, we need extra page in prexor to store the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064) * old data. This page is stored in dev->orig_page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) * This function checks whether we have data for prexor. The exact logic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067) * is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068) * R5_UPTODATE && (!R5_InJournal || R5_OrigPageUPTDODATE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070) static inline bool uptodate_for_rmw(struct r5dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072) return (test_bit(R5_UPTODATE, &dev->flags)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073) (!test_bit(R5_InJournal, &dev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074) test_bit(R5_OrigPageUPTDODATE, &dev->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077) static int handle_stripe_dirtying(struct r5conf *conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078) struct stripe_head *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079) struct stripe_head_state *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080) int disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082) int rmw = 0, rcw = 0, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083) sector_t recovery_cp = conf->mddev->recovery_cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085) /* Check whether resync is now happening or should start.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086) * If yes, then the array is dirty (after unclean shutdown or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087) * initial creation), so parity in some stripes might be inconsistent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) * In this case, we need to always do reconstruct-write, to ensure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089) * that in case of drive failure or read-error correction, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) * generate correct data from the parity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092) if (conf->rmw_level == PARITY_DISABLE_RMW ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093) (recovery_cp < MaxSector && sh->sector >= recovery_cp &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094) s->failed == 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095) /* Calculate the real rcw later - for now make it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096) * look like rcw is cheaper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098) rcw = 1; rmw = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099) pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100) conf->rmw_level, (unsigned long long)recovery_cp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101) (unsigned long long)sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102) } else for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103) /* would I have to read this buffer for read_modify_write */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105) if (((dev->towrite && !delay_towrite(conf, dev, s)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106) i == sh->pd_idx || i == sh->qd_idx ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107) test_bit(R5_InJournal, &dev->flags)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108) !test_bit(R5_LOCKED, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109) !(uptodate_for_rmw(dev) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110) test_bit(R5_Wantcompute, &dev->flags))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111) if (test_bit(R5_Insync, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112) rmw++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114) rmw += 2*disks; /* cannot read it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116) /* Would I have to read this buffer for reconstruct_write */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117) if (!test_bit(R5_OVERWRITE, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118) i != sh->pd_idx && i != sh->qd_idx &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) !test_bit(R5_LOCKED, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120) !(test_bit(R5_UPTODATE, &dev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121) test_bit(R5_Wantcompute, &dev->flags))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) if (test_bit(R5_Insync, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123) rcw++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) rcw += 2*disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129) pr_debug("for sector %llu state 0x%lx, rmw=%d rcw=%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) (unsigned long long)sh->sector, sh->state, rmw, rcw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132) if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_PREFER_RMW)) && rmw > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133) /* prefer read-modify-write, but need to get some data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) if (conf->mddev->queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135) blk_add_trace_msg(conf->mddev->queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) "raid5 rmw %llu %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137) (unsigned long long)sh->sector, rmw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140) if (test_bit(R5_InJournal, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141) dev->page == dev->orig_page &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142) !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143) /* alloc page for prexor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144) struct page *p = alloc_page(GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147) dev->orig_page = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152) * alloc_page() failed, try use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153) * disk_info->extra_page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) if (!test_and_set_bit(R5C_EXTRA_PAGE_IN_USE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156) &conf->cache_state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157) r5c_use_extra_page(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4158) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4161) /* extra_page in use, add to delayed_list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4162) set_bit(STRIPE_DELAYED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4163) s->waiting_extra_page = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4164) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4168) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4169) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4170) if (((dev->towrite && !delay_towrite(conf, dev, s)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4171) i == sh->pd_idx || i == sh->qd_idx ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4172) test_bit(R5_InJournal, &dev->flags)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4173) !test_bit(R5_LOCKED, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4174) !(uptodate_for_rmw(dev) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4175) test_bit(R5_Wantcompute, &dev->flags)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4176) test_bit(R5_Insync, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4177) if (test_bit(STRIPE_PREREAD_ACTIVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4178) &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4179) pr_debug("Read_old block %d for r-m-w\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4180) i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4181) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4182) set_bit(R5_Wantread, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4183) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4184) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4185) set_bit(STRIPE_DELAYED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4189) if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_PREFER_RMW)) && rcw > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4190) /* want reconstruct write, but need to get some data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4191) int qread =0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4192) rcw = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4193) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4194) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4195) if (!test_bit(R5_OVERWRITE, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4196) i != sh->pd_idx && i != sh->qd_idx &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4197) !test_bit(R5_LOCKED, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4198) !(test_bit(R5_UPTODATE, &dev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4199) test_bit(R5_Wantcompute, &dev->flags))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4200) rcw++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4201) if (test_bit(R5_Insync, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4202) test_bit(STRIPE_PREREAD_ACTIVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4203) &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4204) pr_debug("Read_old block "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4205) "%d for Reconstruct\n", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4206) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4207) set_bit(R5_Wantread, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4208) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4209) qread++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4210) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4211) set_bit(STRIPE_DELAYED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4214) if (rcw && conf->mddev->queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4215) blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4216) (unsigned long long)sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4217) rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4220) if (rcw > disks && rmw > disks &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4221) !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4222) set_bit(STRIPE_DELAYED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4224) /* now if nothing is locked, and if we have enough data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4225) * we can start a write request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4227) /* since handle_stripe can be called at any time we need to handle the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4228) * case where a compute block operation has been submitted and then a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4229) * subsequent call wants to start a write request. raid_run_ops only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4230) * handles the case where compute block and reconstruct are requested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4231) * simultaneously. If this is not the case then new writes need to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4232) * held off until the compute completes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4233) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4234) if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4235) (s->locked == 0 && (rcw == 0 || rmw == 0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4236) !test_bit(STRIPE_BIT_DELAY, &sh->state)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4237) schedule_reconstruction(sh, s, rcw == 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4238) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4241) static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4242) struct stripe_head_state *s, int disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4243) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4244) struct r5dev *dev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4246) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4247) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4249) switch (sh->check_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4250) case check_state_idle:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4251) /* start a new check operation if there are no failures */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4252) if (s->failed == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4253) BUG_ON(s->uptodate != disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4254) sh->check_state = check_state_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4255) set_bit(STRIPE_OP_CHECK, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4256) clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4257) s->uptodate--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4258) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4260) dev = &sh->dev[s->failed_num[0]];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4261) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4262) case check_state_compute_result:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4263) sh->check_state = check_state_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4264) if (!dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4265) dev = &sh->dev[sh->pd_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4267) /* check that a write has not made the stripe insync */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4268) if (test_bit(STRIPE_INSYNC, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4269) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4271) /* either failed parity check, or recovery is happening */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4272) BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4273) BUG_ON(s->uptodate != disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4275) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4276) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4277) set_bit(R5_Wantwrite, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4279) clear_bit(STRIPE_DEGRADED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4280) set_bit(STRIPE_INSYNC, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4281) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4282) case check_state_run:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4283) break; /* we will be called again upon completion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4284) case check_state_check_result:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4285) sh->check_state = check_state_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4287) /* if a failure occurred during the check operation, leave
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4288) * STRIPE_INSYNC not set and let the stripe be handled again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4289) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4290) if (s->failed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4291) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4293) /* handle a successful check operation, if parity is correct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4294) * we are done. Otherwise update the mismatch count and repair
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4295) * parity if !MD_RECOVERY_CHECK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4296) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4297) if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4298) /* parity is correct (on disc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4299) * not in buffer any more)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4300) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4301) set_bit(STRIPE_INSYNC, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4302) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4303) atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4304) if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4305) /* don't try to repair!! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4306) set_bit(STRIPE_INSYNC, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4307) pr_warn_ratelimited("%s: mismatch sector in range "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4308) "%llu-%llu\n", mdname(conf->mddev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4309) (unsigned long long) sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4310) (unsigned long long) sh->sector +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4311) RAID5_STRIPE_SECTORS(conf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4312) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4313) sh->check_state = check_state_compute_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4314) set_bit(STRIPE_COMPUTE_RUN, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4315) set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4316) set_bit(R5_Wantcompute,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4317) &sh->dev[sh->pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4318) sh->ops.target = sh->pd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4319) sh->ops.target2 = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4320) s->uptodate++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4323) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4324) case check_state_compute_run:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4325) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4326) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4327) pr_err("%s: unknown check_state: %d sector: %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4328) __func__, sh->check_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4329) (unsigned long long) sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4330) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4334) static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4335) struct stripe_head_state *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4336) int disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4338) int pd_idx = sh->pd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4339) int qd_idx = sh->qd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4340) struct r5dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4342) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4343) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4345) BUG_ON(s->failed > 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4347) /* Want to check and possibly repair P and Q.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4348) * However there could be one 'failed' device, in which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4349) * case we can only check one of them, possibly using the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4350) * other to generate missing data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4353) switch (sh->check_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4354) case check_state_idle:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4355) /* start a new check operation if there are < 2 failures */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4356) if (s->failed == s->q_failed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4357) /* The only possible failed device holds Q, so it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4358) * makes sense to check P (If anything else were failed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4359) * we would have used P to recreate it).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4360) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4361) sh->check_state = check_state_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4363) if (!s->q_failed && s->failed < 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4364) /* Q is not failed, and we didn't use it to generate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4365) * anything, so it makes sense to check it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4366) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4367) if (sh->check_state == check_state_run)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4368) sh->check_state = check_state_run_pq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4369) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4370) sh->check_state = check_state_run_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4373) /* discard potentially stale zero_sum_result */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4374) sh->ops.zero_sum_result = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4376) if (sh->check_state == check_state_run) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4377) /* async_xor_zero_sum destroys the contents of P */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4378) clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4379) s->uptodate--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4381) if (sh->check_state >= check_state_run &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4382) sh->check_state <= check_state_run_pq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4383) /* async_syndrome_zero_sum preserves P and Q, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4384) * no need to mark them !uptodate here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4385) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4386) set_bit(STRIPE_OP_CHECK, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4387) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4390) /* we have 2-disk failure */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4391) BUG_ON(s->failed != 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4392) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4393) case check_state_compute_result:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4394) sh->check_state = check_state_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4396) /* check that a write has not made the stripe insync */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4397) if (test_bit(STRIPE_INSYNC, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4398) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4400) /* now write out any block on a failed drive,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4401) * or P or Q if they were recomputed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4402) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4403) dev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4404) if (s->failed == 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4405) dev = &sh->dev[s->failed_num[1]];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4406) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4407) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4408) set_bit(R5_Wantwrite, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4410) if (s->failed >= 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4411) dev = &sh->dev[s->failed_num[0]];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4412) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4413) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4414) set_bit(R5_Wantwrite, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4416) if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4417) dev = &sh->dev[pd_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4418) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4419) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4420) set_bit(R5_Wantwrite, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4422) if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4423) dev = &sh->dev[qd_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4424) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4425) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4426) set_bit(R5_Wantwrite, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4428) if (WARN_ONCE(dev && !test_bit(R5_UPTODATE, &dev->flags),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4429) "%s: disk%td not up to date\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4430) mdname(conf->mddev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4431) dev - (struct r5dev *) &sh->dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4432) clear_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4433) clear_bit(R5_Wantwrite, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4434) s->locked--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4436) clear_bit(STRIPE_DEGRADED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4438) set_bit(STRIPE_INSYNC, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4439) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4440) case check_state_run:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4441) case check_state_run_q:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4442) case check_state_run_pq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4443) break; /* we will be called again upon completion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4444) case check_state_check_result:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4445) sh->check_state = check_state_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4447) /* handle a successful check operation, if parity is correct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4448) * we are done. Otherwise update the mismatch count and repair
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4449) * parity if !MD_RECOVERY_CHECK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4450) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4451) if (sh->ops.zero_sum_result == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4452) /* both parities are correct */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4453) if (!s->failed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4454) set_bit(STRIPE_INSYNC, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4455) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4456) /* in contrast to the raid5 case we can validate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4457) * parity, but still have a failure to write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4458) * back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4459) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4460) sh->check_state = check_state_compute_result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4461) /* Returning at this point means that we may go
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4462) * off and bring p and/or q uptodate again so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4463) * we make sure to check zero_sum_result again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4464) * to verify if p or q need writeback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4465) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4467) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4468) atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4469) if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4470) /* don't try to repair!! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4471) set_bit(STRIPE_INSYNC, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4472) pr_warn_ratelimited("%s: mismatch sector in range "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4473) "%llu-%llu\n", mdname(conf->mddev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4474) (unsigned long long) sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4475) (unsigned long long) sh->sector +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4476) RAID5_STRIPE_SECTORS(conf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4477) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4478) int *target = &sh->ops.target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4480) sh->ops.target = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4481) sh->ops.target2 = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4482) sh->check_state = check_state_compute_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4483) set_bit(STRIPE_COMPUTE_RUN, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4484) set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4485) if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4486) set_bit(R5_Wantcompute,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4487) &sh->dev[pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4488) *target = pd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4489) target = &sh->ops.target2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4490) s->uptodate++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4492) if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4493) set_bit(R5_Wantcompute,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4494) &sh->dev[qd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4495) *target = qd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4496) s->uptodate++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4500) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4501) case check_state_compute_run:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4502) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4503) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4504) pr_warn("%s: unknown check_state: %d sector: %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4505) __func__, sh->check_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4506) (unsigned long long) sh->sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4507) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4511) static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4512) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4513) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4515) /* We have read all the blocks in this stripe and now we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4516) * copy some of them into a target stripe for expand.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4517) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4518) struct dma_async_tx_descriptor *tx = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4519) BUG_ON(sh->batch_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4520) clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4521) for (i = 0; i < sh->disks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4522) if (i != sh->pd_idx && i != sh->qd_idx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4523) int dd_idx, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4524) struct stripe_head *sh2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4525) struct async_submit_ctl submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4527) sector_t bn = raid5_compute_blocknr(sh, i, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4528) sector_t s = raid5_compute_sector(conf, bn, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4529) &dd_idx, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4530) sh2 = raid5_get_active_stripe(conf, s, 0, 1, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4531) if (sh2 == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4532) /* so far only the early blocks of this stripe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4533) * have been requested. When later blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4534) * get requested, we will try again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4535) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4536) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4537) if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4538) test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4539) /* must have already done this block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4540) raid5_release_stripe(sh2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4541) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4544) /* place all the copies on one channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4545) init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4546) tx = async_memcpy(sh2->dev[dd_idx].page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4547) sh->dev[i].page, sh2->dev[dd_idx].offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4548) sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4549) &submit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4551) set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4552) set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4553) for (j = 0; j < conf->raid_disks; j++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4554) if (j != sh2->pd_idx &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4555) j != sh2->qd_idx &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4556) !test_bit(R5_Expanded, &sh2->dev[j].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4557) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4558) if (j == conf->raid_disks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4559) set_bit(STRIPE_EXPAND_READY, &sh2->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4560) set_bit(STRIPE_HANDLE, &sh2->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4562) raid5_release_stripe(sh2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4565) /* done submitting copies, wait for them to complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4566) async_tx_quiesce(&tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4569) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4570) * handle_stripe - do things to a stripe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4571) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4572) * We lock the stripe by setting STRIPE_ACTIVE and then examine the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4573) * state of various bits to see what needs to be done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4574) * Possible results:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4575) * return some read requests which now have data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4576) * return some write requests which are safely on storage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4577) * schedule a read on some buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4578) * schedule a write of some buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4579) * return confirmation of parity correctness
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4580) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4581) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4583) static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4584) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4585) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4586) int disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4587) struct r5dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4588) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4589) int do_recovery = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4591) memset(s, 0, sizeof(*s));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4593) s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4594) s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4595) s->failed_num[0] = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4596) s->failed_num[1] = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4597) s->log_failed = r5l_log_disk_error(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4599) /* Now to look around and see what can be done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4600) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4601) for (i=disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4602) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4603) sector_t first_bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4604) int bad_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4605) int is_bad = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4607) dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4609) pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4610) i, dev->flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4611) dev->toread, dev->towrite, dev->written);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4612) /* maybe we can reply to a read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4613) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4614) * new wantfill requests are only permitted while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4615) * ops_complete_biofill is guaranteed to be inactive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4616) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4617) if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4618) !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4619) set_bit(R5_Wantfill, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4621) /* now count some things */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4622) if (test_bit(R5_LOCKED, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4623) s->locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4624) if (test_bit(R5_UPTODATE, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4625) s->uptodate++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4626) if (test_bit(R5_Wantcompute, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4627) s->compute++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4628) BUG_ON(s->compute > 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4631) if (test_bit(R5_Wantfill, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4632) s->to_fill++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4633) else if (dev->toread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4634) s->to_read++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4635) if (dev->towrite) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4636) s->to_write++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4637) if (!test_bit(R5_OVERWRITE, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4638) s->non_overwrite++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4640) if (dev->written)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4641) s->written++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4642) /* Prefer to use the replacement for reads, but only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4643) * if it is recovered enough and has no bad blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4644) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4645) rdev = rcu_dereference(conf->disks[i].replacement);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4646) if (rdev && !test_bit(Faulty, &rdev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4647) rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4648) !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4649) &first_bad, &bad_sectors))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4650) set_bit(R5_ReadRepl, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4651) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4652) if (rdev && !test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4653) set_bit(R5_NeedReplace, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4654) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4655) clear_bit(R5_NeedReplace, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4656) rdev = rcu_dereference(conf->disks[i].rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4657) clear_bit(R5_ReadRepl, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4659) if (rdev && test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4660) rdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4661) if (rdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4662) is_bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4663) &first_bad, &bad_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4664) if (s->blocked_rdev == NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4665) && (test_bit(Blocked, &rdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4666) || is_bad < 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4667) if (is_bad < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4668) set_bit(BlockedBadBlocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4669) &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4670) s->blocked_rdev = rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4671) atomic_inc(&rdev->nr_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4674) clear_bit(R5_Insync, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4675) if (!rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4676) /* Not in-sync */;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4677) else if (is_bad) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4678) /* also not in-sync */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4679) if (!test_bit(WriteErrorSeen, &rdev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4680) test_bit(R5_UPTODATE, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4681) /* treat as in-sync, but with a read error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4682) * which we can now try to correct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4683) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4684) set_bit(R5_Insync, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4685) set_bit(R5_ReadError, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4687) } else if (test_bit(In_sync, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4688) set_bit(R5_Insync, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4689) else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4690) /* in sync if before recovery_offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4691) set_bit(R5_Insync, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4692) else if (test_bit(R5_UPTODATE, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4693) test_bit(R5_Expanded, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4694) /* If we've reshaped into here, we assume it is Insync.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4695) * We will shortly update recovery_offset to make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4696) * it official.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4697) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4698) set_bit(R5_Insync, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4700) if (test_bit(R5_WriteError, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4701) /* This flag does not apply to '.replacement'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4702) * only to .rdev, so make sure to check that*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4703) struct md_rdev *rdev2 = rcu_dereference(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4704) conf->disks[i].rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4705) if (rdev2 == rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4706) clear_bit(R5_Insync, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4707) if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4708) s->handle_bad_blocks = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4709) atomic_inc(&rdev2->nr_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4710) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4711) clear_bit(R5_WriteError, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4713) if (test_bit(R5_MadeGood, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4714) /* This flag does not apply to '.replacement'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4715) * only to .rdev, so make sure to check that*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4716) struct md_rdev *rdev2 = rcu_dereference(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4717) conf->disks[i].rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4718) if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4719) s->handle_bad_blocks = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4720) atomic_inc(&rdev2->nr_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4721) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4722) clear_bit(R5_MadeGood, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4724) if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4725) struct md_rdev *rdev2 = rcu_dereference(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4726) conf->disks[i].replacement);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4727) if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4728) s->handle_bad_blocks = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4729) atomic_inc(&rdev2->nr_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4730) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4731) clear_bit(R5_MadeGoodRepl, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4733) if (!test_bit(R5_Insync, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4734) /* The ReadError flag will just be confusing now */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4735) clear_bit(R5_ReadError, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4736) clear_bit(R5_ReWrite, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4738) if (test_bit(R5_ReadError, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4739) clear_bit(R5_Insync, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4740) if (!test_bit(R5_Insync, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4741) if (s->failed < 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4742) s->failed_num[s->failed] = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4743) s->failed++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4744) if (rdev && !test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4745) do_recovery = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4746) else if (!rdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4747) rdev = rcu_dereference(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4748) conf->disks[i].replacement);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4749) if (rdev && !test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4750) do_recovery = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4754) if (test_bit(R5_InJournal, &dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4755) s->injournal++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4756) if (test_bit(R5_InJournal, &dev->flags) && dev->written)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4757) s->just_cached++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4759) if (test_bit(STRIPE_SYNCING, &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4760) /* If there is a failed device being replaced,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4761) * we must be recovering.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4762) * else if we are after recovery_cp, we must be syncing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4763) * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4764) * else we can only be replacing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4765) * sync and recovery both need to read all devices, and so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4766) * use the same flag.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4767) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4768) if (do_recovery ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4769) sh->sector >= conf->mddev->recovery_cp ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4770) test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4771) s->syncing = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4772) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4773) s->replacing = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4775) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4778) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4779) * Return '1' if this is a member of batch, or '0' if it is a lone stripe or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4780) * a head which can now be handled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4781) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4782) static int clear_batch_ready(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4783) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4784) struct stripe_head *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4785) if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4786) return (sh->batch_head && sh->batch_head != sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4787) spin_lock(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4788) if (!sh->batch_head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4789) spin_unlock(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4790) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4791) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4793) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4794) * this stripe could be added to a batch list before we check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4795) * BATCH_READY, skips it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4796) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4797) if (sh->batch_head != sh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4798) spin_unlock(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4799) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4801) spin_lock(&sh->batch_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4802) list_for_each_entry(tmp, &sh->batch_list, batch_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4803) clear_bit(STRIPE_BATCH_READY, &tmp->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4804) spin_unlock(&sh->batch_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4805) spin_unlock(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4807) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4808) * BATCH_READY is cleared, no new stripes can be added.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4809) * batch_list can be accessed without lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4810) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4811) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4814) static void break_stripe_batch_list(struct stripe_head *head_sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4815) unsigned long handle_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4816) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4817) struct stripe_head *sh, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4818) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4819) int do_wakeup = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4821) list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4823) list_del_init(&sh->batch_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4825) WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4826) (1 << STRIPE_SYNCING) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4827) (1 << STRIPE_REPLACED) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4828) (1 << STRIPE_DELAYED) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4829) (1 << STRIPE_BIT_DELAY) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4830) (1 << STRIPE_FULL_WRITE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4831) (1 << STRIPE_BIOFILL_RUN) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4832) (1 << STRIPE_COMPUTE_RUN) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4833) (1 << STRIPE_DISCARD) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4834) (1 << STRIPE_BATCH_READY) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4835) (1 << STRIPE_BATCH_ERR) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4836) (1 << STRIPE_BITMAP_PENDING)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4837) "stripe state: %lx\n", sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4838) WARN_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4839) (1 << STRIPE_REPLACED)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4840) "head stripe state: %lx\n", head_sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4842) set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4843) (1 << STRIPE_PREREAD_ACTIVE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4844) (1 << STRIPE_DEGRADED) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4845) (1 << STRIPE_ON_UNPLUG_LIST)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4846) head_sh->state & (1 << STRIPE_INSYNC));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4848) sh->check_state = head_sh->check_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4849) sh->reconstruct_state = head_sh->reconstruct_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4850) spin_lock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4851) sh->batch_head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4852) spin_unlock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4853) for (i = 0; i < sh->disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4854) if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4855) do_wakeup = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4856) sh->dev[i].flags = head_sh->dev[i].flags &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4857) (~((1 << R5_WriteError) | (1 << R5_Overlap)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4858) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4859) if (handle_flags == 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4860) sh->state & handle_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4861) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4862) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4864) spin_lock_irq(&head_sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4865) head_sh->batch_head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4866) spin_unlock_irq(&head_sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4867) for (i = 0; i < head_sh->disks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4868) if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4869) do_wakeup = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4870) if (head_sh->state & handle_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4871) set_bit(STRIPE_HANDLE, &head_sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4873) if (do_wakeup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4874) wake_up(&head_sh->raid_conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4877) static void handle_stripe(struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4878) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4879) struct stripe_head_state s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4880) struct r5conf *conf = sh->raid_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4881) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4882) int prexor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4883) int disks = sh->disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4884) struct r5dev *pdev, *qdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4886) clear_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4888) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4889) * handle_stripe should not continue handle the batched stripe, only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4890) * the head of batch list or lone stripe can continue. Otherwise we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4891) * could see break_stripe_batch_list warns about the STRIPE_ACTIVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4892) * is set for the batched stripe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4893) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4894) if (clear_batch_ready(sh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4895) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4897) if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4898) /* already being handled, ensure it gets handled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4899) * again when current action finishes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4900) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4901) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4904) if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4905) break_stripe_batch_list(sh, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4907) if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4908) spin_lock(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4909) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4910) * Cannot process 'sync' concurrently with 'discard'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4911) * Flush data in r5cache before 'sync'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4912) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4913) if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4914) !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4915) !test_bit(STRIPE_DISCARD, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4916) test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4917) set_bit(STRIPE_SYNCING, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4918) clear_bit(STRIPE_INSYNC, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4919) clear_bit(STRIPE_REPLACED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4921) spin_unlock(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4922) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4923) clear_bit(STRIPE_DELAYED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4925) pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4926) "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4927) (unsigned long long)sh->sector, sh->state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4928) atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4929) sh->check_state, sh->reconstruct_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4931) analyse_stripe(sh, &s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4933) if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4934) goto finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4936) if (s.handle_bad_blocks ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4937) test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4938) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4939) goto finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4942) if (unlikely(s.blocked_rdev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4943) if (s.syncing || s.expanding || s.expanded ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4944) s.replacing || s.to_write || s.written) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4945) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4946) goto finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4947) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4948) /* There is nothing for the blocked_rdev to block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4949) rdev_dec_pending(s.blocked_rdev, conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4950) s.blocked_rdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4953) if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4954) set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4955) set_bit(STRIPE_BIOFILL_RUN, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4958) pr_debug("locked=%d uptodate=%d to_read=%d"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4959) " to_write=%d failed=%d failed_num=%d,%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4960) s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4961) s.failed_num[0], s.failed_num[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4962) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4963) * check if the array has lost more than max_degraded devices and,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4964) * if so, some requests might need to be failed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4965) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4966) * When journal device failed (log_failed), we will only process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4967) * the stripe if there is data need write to raid disks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4968) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4969) if (s.failed > conf->max_degraded ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4970) (s.log_failed && s.injournal == 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4971) sh->check_state = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4972) sh->reconstruct_state = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4973) break_stripe_batch_list(sh, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4974) if (s.to_read+s.to_write+s.written)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4975) handle_failed_stripe(conf, sh, &s, disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4976) if (s.syncing + s.replacing)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4977) handle_failed_sync(conf, sh, &s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4980) /* Now we check to see if any write operations have recently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4981) * completed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4982) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4983) prexor = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4984) if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4985) prexor = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4986) if (sh->reconstruct_state == reconstruct_state_drain_result ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4987) sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4988) sh->reconstruct_state = reconstruct_state_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4990) /* All the 'written' buffers and the parity block are ready to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4991) * be written back to disk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4992) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4993) BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4994) !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4995) BUG_ON(sh->qd_idx >= 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4996) !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4997) !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4998) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4999) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5000) if (test_bit(R5_LOCKED, &dev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5001) (i == sh->pd_idx || i == sh->qd_idx ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5002) dev->written || test_bit(R5_InJournal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5003) &dev->flags))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5004) pr_debug("Writing block %d\n", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5005) set_bit(R5_Wantwrite, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5006) if (prexor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5007) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5008) if (s.failed > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5009) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5010) if (!test_bit(R5_Insync, &dev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5011) ((i == sh->pd_idx || i == sh->qd_idx) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5012) s.failed == 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5013) set_bit(STRIPE_INSYNC, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5016) if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5017) s.dec_preread_active = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5018) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5020) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5021) * might be able to return some write requests if the parity blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5022) * are safe, or on a failed drive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5023) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5024) pdev = &sh->dev[sh->pd_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5025) s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5026) || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5027) qdev = &sh->dev[sh->qd_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5028) s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5029) || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5030) || conf->level < 6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5032) if (s.written &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5033) (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5034) && !test_bit(R5_LOCKED, &pdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5035) && (test_bit(R5_UPTODATE, &pdev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5036) test_bit(R5_Discard, &pdev->flags))))) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5037) (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5038) && !test_bit(R5_LOCKED, &qdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5039) && (test_bit(R5_UPTODATE, &qdev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5040) test_bit(R5_Discard, &qdev->flags))))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5041) handle_stripe_clean_event(conf, sh, disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5043) if (s.just_cached)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5044) r5c_handle_cached_data_endio(conf, sh, disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5045) log_stripe_write_finished(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5047) /* Now we might consider reading some blocks, either to check/generate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5048) * parity, or to satisfy requests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5049) * or to load a block that is being partially written.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5050) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5051) if (s.to_read || s.non_overwrite
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5052) || (s.to_write && s.failed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5053) || (s.syncing && (s.uptodate + s.compute < disks))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5054) || s.replacing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5055) || s.expanding)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5056) handle_stripe_fill(sh, &s, disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5058) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5059) * When the stripe finishes full journal write cycle (write to journal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5060) * and raid disk), this is the clean up procedure so it is ready for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5061) * next operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5062) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5063) r5c_finish_stripe_write_out(conf, sh, &s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5065) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5066) * Now to consider new write requests, cache write back and what else,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5067) * if anything should be read. We do not handle new writes when:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5068) * 1/ A 'write' operation (copy+xor) is already in flight.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5069) * 2/ A 'check' operation is in flight, as it may clobber the parity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5070) * block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5071) * 3/ A r5c cache log write is in flight.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5072) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5074) if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5075) if (!r5c_is_writeback(conf->log)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5076) if (s.to_write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5077) handle_stripe_dirtying(conf, sh, &s, disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5078) } else { /* write back cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5079) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5081) /* First, try handle writes in caching phase */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5082) if (s.to_write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5083) ret = r5c_try_caching_write(conf, sh, &s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5084) disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5085) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5086) * If caching phase failed: ret == -EAGAIN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5087) * OR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5088) * stripe under reclaim: !caching && injournal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5089) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5090) * fall back to handle_stripe_dirtying()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5091) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5092) if (ret == -EAGAIN ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5093) /* stripe under reclaim: !caching && injournal */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5094) (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5095) s.injournal > 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5096) ret = handle_stripe_dirtying(conf, sh, &s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5097) disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5098) if (ret == -EAGAIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5099) goto finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5104) /* maybe we need to check and possibly fix the parity for this stripe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5105) * Any reads will already have been scheduled, so we just see if enough
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5106) * data is available. The parity check is held off while parity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5107) * dependent operations are in flight.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5109) if (sh->check_state ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5110) (s.syncing && s.locked == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5111) !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5112) !test_bit(STRIPE_INSYNC, &sh->state))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5113) if (conf->level == 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5114) handle_parity_checks6(conf, sh, &s, disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5115) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5116) handle_parity_checks5(conf, sh, &s, disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5119) if ((s.replacing || s.syncing) && s.locked == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5120) && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5121) && !test_bit(STRIPE_REPLACED, &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5122) /* Write out to replacement devices where possible */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5123) for (i = 0; i < conf->raid_disks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5124) if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5125) WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5126) set_bit(R5_WantReplace, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5127) set_bit(R5_LOCKED, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5128) s.locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5130) if (s.replacing)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5131) set_bit(STRIPE_INSYNC, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5132) set_bit(STRIPE_REPLACED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5134) if ((s.syncing || s.replacing) && s.locked == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5135) !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5136) test_bit(STRIPE_INSYNC, &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5137) md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5138) clear_bit(STRIPE_SYNCING, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5139) if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5140) wake_up(&conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5143) /* If the failed drives are just a ReadError, then we might need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5144) * to progress the repair/check process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5145) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5146) if (s.failed <= conf->max_degraded && !conf->mddev->ro)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5147) for (i = 0; i < s.failed; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5148) struct r5dev *dev = &sh->dev[s.failed_num[i]];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5149) if (test_bit(R5_ReadError, &dev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5150) && !test_bit(R5_LOCKED, &dev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5151) && test_bit(R5_UPTODATE, &dev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5152) ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5153) if (!test_bit(R5_ReWrite, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5154) set_bit(R5_Wantwrite, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5155) set_bit(R5_ReWrite, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5156) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5157) /* let's read it back */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5158) set_bit(R5_Wantread, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5159) set_bit(R5_LOCKED, &dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5160) s.locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5164) /* Finish reconstruct operations initiated by the expansion process */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5165) if (sh->reconstruct_state == reconstruct_state_result) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5166) struct stripe_head *sh_src
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5167) = raid5_get_active_stripe(conf, sh->sector, 1, 1, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5168) if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5169) /* sh cannot be written until sh_src has been read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5170) * so arrange for sh to be delayed a little
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5171) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5172) set_bit(STRIPE_DELAYED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5173) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5174) if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5175) &sh_src->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5176) atomic_inc(&conf->preread_active_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5177) raid5_release_stripe(sh_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5178) goto finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5180) if (sh_src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5181) raid5_release_stripe(sh_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5183) sh->reconstruct_state = reconstruct_state_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5184) clear_bit(STRIPE_EXPANDING, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5185) for (i = conf->raid_disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5186) set_bit(R5_Wantwrite, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5187) set_bit(R5_LOCKED, &sh->dev[i].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5188) s.locked++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5192) if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5193) !sh->reconstruct_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5194) /* Need to write out all blocks after computing parity */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5195) sh->disks = conf->raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5196) stripe_set_idx(sh->sector, conf, 0, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5197) schedule_reconstruction(sh, &s, 1, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5198) } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5199) clear_bit(STRIPE_EXPAND_READY, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5200) atomic_dec(&conf->reshape_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5201) wake_up(&conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5202) md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5205) if (s.expanding && s.locked == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5206) !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5207) handle_stripe_expansion(conf, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5209) finish:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5210) /* wait for this device to become unblocked */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5211) if (unlikely(s.blocked_rdev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5212) if (conf->mddev->external)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5213) md_wait_for_blocked_rdev(s.blocked_rdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5214) conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5215) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5216) /* Internal metadata will immediately
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5217) * be written by raid5d, so we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5218) * need to wait here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5219) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5220) rdev_dec_pending(s.blocked_rdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5221) conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5224) if (s.handle_bad_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5225) for (i = disks; i--; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5226) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5227) struct r5dev *dev = &sh->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5228) if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5229) /* We own a safe reference to the rdev */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5230) rdev = conf->disks[i].rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5231) if (!rdev_set_badblocks(rdev, sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5232) RAID5_STRIPE_SECTORS(conf), 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5233) md_error(conf->mddev, rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5234) rdev_dec_pending(rdev, conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5236) if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5237) rdev = conf->disks[i].rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5238) rdev_clear_badblocks(rdev, sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5239) RAID5_STRIPE_SECTORS(conf), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5240) rdev_dec_pending(rdev, conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5242) if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5243) rdev = conf->disks[i].replacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5244) if (!rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5245) /* rdev have been moved down */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5246) rdev = conf->disks[i].rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5247) rdev_clear_badblocks(rdev, sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5248) RAID5_STRIPE_SECTORS(conf), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5249) rdev_dec_pending(rdev, conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5253) if (s.ops_request)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5254) raid_run_ops(sh, s.ops_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5256) ops_run_io(sh, &s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5258) if (s.dec_preread_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5259) /* We delay this until after ops_run_io so that if make_request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5260) * is waiting on a flush, it won't continue until the writes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5261) * have actually been submitted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5262) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5263) atomic_dec(&conf->preread_active_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5264) if (atomic_read(&conf->preread_active_stripes) <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5265) IO_THRESHOLD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5266) md_wakeup_thread(conf->mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5269) clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5272) static void raid5_activate_delayed(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5274) if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5275) while (!list_empty(&conf->delayed_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5276) struct list_head *l = conf->delayed_list.next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5277) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5278) sh = list_entry(l, struct stripe_head, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5279) list_del_init(l);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5280) clear_bit(STRIPE_DELAYED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5281) if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5282) atomic_inc(&conf->preread_active_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5283) list_add_tail(&sh->lru, &conf->hold_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5284) raid5_wakeup_stripe_thread(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5289) static void activate_bit_delay(struct r5conf *conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5290) struct list_head *temp_inactive_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5292) /* device_lock is held */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5293) struct list_head head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5294) list_add(&head, &conf->bitmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5295) list_del_init(&conf->bitmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5296) while (!list_empty(&head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5297) struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5298) int hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5299) list_del_init(&sh->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5300) atomic_inc(&sh->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5301) hash = sh->hash_lock_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5302) __release_stripe(conf, sh, &temp_inactive_list[hash]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5306) static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5308) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5309) sector_t sector = bio->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5310) unsigned int chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5311) unsigned int bio_sectors = bio_sectors(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5313) WARN_ON_ONCE(bio->bi_partno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5315) chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5316) return chunk_sectors >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5317) ((sector & (chunk_sectors - 1)) + bio_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5320) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5321) * add bio to the retry LIFO ( in O(1) ... we are in interrupt )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5322) * later sampled by raid5d.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5323) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5324) static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5326) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5328) spin_lock_irqsave(&conf->device_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5330) bi->bi_next = conf->retry_read_aligned_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5331) conf->retry_read_aligned_list = bi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5333) spin_unlock_irqrestore(&conf->device_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5334) md_wakeup_thread(conf->mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5337) static struct bio *remove_bio_from_retry(struct r5conf *conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5338) unsigned int *offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5339) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5340) struct bio *bi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5342) bi = conf->retry_read_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5343) if (bi) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5344) *offset = conf->retry_read_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5345) conf->retry_read_aligned = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5346) return bi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5348) bi = conf->retry_read_aligned_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5349) if(bi) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5350) conf->retry_read_aligned_list = bi->bi_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5351) bi->bi_next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5352) *offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5355) return bi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5358) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5359) * The "raid5_align_endio" should check if the read succeeded and if it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5360) * did, call bio_endio on the original bio (having bio_put the new bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5361) * first).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5362) * If the read failed..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5363) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5364) static void raid5_align_endio(struct bio *bi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5366) struct bio* raid_bi = bi->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5367) struct mddev *mddev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5368) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5369) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5370) blk_status_t error = bi->bi_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5372) bio_put(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5374) rdev = (void*)raid_bi->bi_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5375) raid_bi->bi_next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5376) mddev = rdev->mddev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5377) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5379) rdev_dec_pending(rdev, conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5381) if (!error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5382) bio_endio(raid_bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5383) if (atomic_dec_and_test(&conf->active_aligned_reads))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5384) wake_up(&conf->wait_for_quiescent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5385) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5388) pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5390) add_bio_to_retry(raid_bi, conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5393) static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5395) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5396) int dd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5397) struct bio* align_bi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5398) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5399) sector_t end_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5401) if (!in_chunk_boundary(mddev, raid_bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5402) pr_debug("%s: non aligned\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5403) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5405) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5406) * use bio_clone_fast to make a copy of the bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5408) align_bi = bio_clone_fast(raid_bio, GFP_NOIO, &mddev->bio_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5409) if (!align_bi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5410) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5411) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5412) * set bi_end_io to a new function, and set bi_private to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5413) * original bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5414) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5415) align_bi->bi_end_io = raid5_align_endio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5416) align_bi->bi_private = raid_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5417) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5418) * compute position
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5419) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5420) align_bi->bi_iter.bi_sector =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5421) raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5422) 0, &dd_idx, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5424) end_sector = bio_end_sector(align_bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5425) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5426) rdev = rcu_dereference(conf->disks[dd_idx].replacement);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5427) if (!rdev || test_bit(Faulty, &rdev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5428) rdev->recovery_offset < end_sector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5429) rdev = rcu_dereference(conf->disks[dd_idx].rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5430) if (rdev &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5431) (test_bit(Faulty, &rdev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5432) !(test_bit(In_sync, &rdev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5433) rdev->recovery_offset >= end_sector)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5434) rdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5437) if (r5c_big_stripe_cached(conf, align_bi->bi_iter.bi_sector)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5438) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5439) bio_put(align_bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5440) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5443) if (rdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5444) sector_t first_bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5445) int bad_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5447) atomic_inc(&rdev->nr_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5448) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5449) raid_bio->bi_next = (void*)rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5450) bio_set_dev(align_bi, rdev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5452) if (is_badblock(rdev, align_bi->bi_iter.bi_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5453) bio_sectors(align_bi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5454) &first_bad, &bad_sectors)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5455) bio_put(align_bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5456) rdev_dec_pending(rdev, mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5457) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5460) /* No reshape active, so we can trust rdev->data_offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5461) align_bi->bi_iter.bi_sector += rdev->data_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5463) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5464) wait_event_lock_irq(conf->wait_for_quiescent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5465) conf->quiesce == 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5466) conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5467) atomic_inc(&conf->active_aligned_reads);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5468) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5470) if (mddev->gendisk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5471) trace_block_bio_remap(align_bi->bi_disk->queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5472) align_bi, disk_devt(mddev->gendisk),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5473) raid_bio->bi_iter.bi_sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5474) submit_bio_noacct(align_bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5475) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5476) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5477) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5478) bio_put(align_bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5479) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5483) static struct bio *chunk_aligned_read(struct mddev *mddev, struct bio *raid_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5484) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5485) struct bio *split;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5486) sector_t sector = raid_bio->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5487) unsigned chunk_sects = mddev->chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5488) unsigned sectors = chunk_sects - (sector & (chunk_sects-1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5490) if (sectors < bio_sectors(raid_bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5491) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5492) split = bio_split(raid_bio, sectors, GFP_NOIO, &conf->bio_split);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5493) bio_chain(split, raid_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5494) submit_bio_noacct(raid_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5495) raid_bio = split;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5498) if (!raid5_read_one_chunk(mddev, raid_bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5499) return raid_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5501) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5504) /* __get_priority_stripe - get the next stripe to process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5505) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5506) * Full stripe writes are allowed to pass preread active stripes up until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5507) * the bypass_threshold is exceeded. In general the bypass_count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5508) * increments when the handle_list is handled before the hold_list; however, it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5509) * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5510) * stripe with in flight i/o. The bypass_count will be reset when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5511) * head of the hold_list has changed, i.e. the head was promoted to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5512) * handle_list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5513) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5514) static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5515) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5516) struct stripe_head *sh, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5517) struct list_head *handle_list = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5518) struct r5worker_group *wg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5519) bool second_try = !r5c_is_writeback(conf->log) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5520) !r5l_log_disk_error(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5521) bool try_loprio = test_bit(R5C_LOG_TIGHT, &conf->cache_state) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5522) r5l_log_disk_error(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5524) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5525) wg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5526) sh = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5527) if (conf->worker_cnt_per_group == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5528) handle_list = try_loprio ? &conf->loprio_list :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5529) &conf->handle_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5530) } else if (group != ANY_GROUP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5531) handle_list = try_loprio ? &conf->worker_groups[group].loprio_list :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5532) &conf->worker_groups[group].handle_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5533) wg = &conf->worker_groups[group];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5534) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5535) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5536) for (i = 0; i < conf->group_cnt; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5537) handle_list = try_loprio ? &conf->worker_groups[i].loprio_list :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5538) &conf->worker_groups[i].handle_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5539) wg = &conf->worker_groups[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5540) if (!list_empty(handle_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5541) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5545) pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5546) __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5547) list_empty(handle_list) ? "empty" : "busy",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5548) list_empty(&conf->hold_list) ? "empty" : "busy",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5549) atomic_read(&conf->pending_full_writes), conf->bypass_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5551) if (!list_empty(handle_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5552) sh = list_entry(handle_list->next, typeof(*sh), lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5554) if (list_empty(&conf->hold_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5555) conf->bypass_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5556) else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5557) if (conf->hold_list.next == conf->last_hold)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5558) conf->bypass_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5559) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5560) conf->last_hold = conf->hold_list.next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5561) conf->bypass_count -= conf->bypass_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5562) if (conf->bypass_count < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5563) conf->bypass_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5566) } else if (!list_empty(&conf->hold_list) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5567) ((conf->bypass_threshold &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5568) conf->bypass_count > conf->bypass_threshold) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5569) atomic_read(&conf->pending_full_writes) == 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5571) list_for_each_entry(tmp, &conf->hold_list, lru) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5572) if (conf->worker_cnt_per_group == 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5573) group == ANY_GROUP ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5574) !cpu_online(tmp->cpu) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5575) cpu_to_group(tmp->cpu) == group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5576) sh = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5577) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5581) if (sh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5582) conf->bypass_count -= conf->bypass_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5583) if (conf->bypass_count < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5584) conf->bypass_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5586) wg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5589) if (!sh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5590) if (second_try)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5591) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5592) second_try = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5593) try_loprio = !try_loprio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5594) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5597) if (wg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5598) wg->stripes_cnt--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5599) sh->group = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5601) list_del_init(&sh->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5602) BUG_ON(atomic_inc_return(&sh->count) != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5603) return sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5604) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5606) struct raid5_plug_cb {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5607) struct blk_plug_cb cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5608) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5609) struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5610) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5612) static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5613) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5614) struct raid5_plug_cb *cb = container_of(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5615) blk_cb, struct raid5_plug_cb, cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5616) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5617) struct mddev *mddev = cb->cb.data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5618) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5619) int cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5620) int hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5622) if (cb->list.next && !list_empty(&cb->list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5623) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5624) while (!list_empty(&cb->list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5625) sh = list_first_entry(&cb->list, struct stripe_head, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5626) list_del_init(&sh->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5627) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5628) * avoid race release_stripe_plug() sees
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5629) * STRIPE_ON_UNPLUG_LIST clear but the stripe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5630) * is still in our list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5631) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5632) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5633) clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5634) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5635) * STRIPE_ON_RELEASE_LIST could be set here. In that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5636) * case, the count is always > 1 here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5637) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5638) hash = sh->hash_lock_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5639) __release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5640) cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5642) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5644) release_inactive_stripe_list(conf, cb->temp_inactive_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5645) NR_STRIPE_HASH_LOCKS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5646) if (mddev->queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5647) trace_block_unplug(mddev->queue, cnt, !from_schedule);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5648) kfree(cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5651) static void release_stripe_plug(struct mddev *mddev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5652) struct stripe_head *sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5654) struct blk_plug_cb *blk_cb = blk_check_plugged(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5655) raid5_unplug, mddev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5656) sizeof(struct raid5_plug_cb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5657) struct raid5_plug_cb *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5659) if (!blk_cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5660) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5661) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5664) cb = container_of(blk_cb, struct raid5_plug_cb, cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5666) if (cb->list.next == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5667) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5668) INIT_LIST_HEAD(&cb->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5669) for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5670) INIT_LIST_HEAD(cb->temp_inactive_list + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5671) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5673) if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5674) list_add_tail(&sh->lru, &cb->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5675) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5676) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5679) static void make_discard_request(struct mddev *mddev, struct bio *bi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5680) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5681) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5682) sector_t logical_sector, last_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5683) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5684) int stripe_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5686) if (mddev->reshape_position != MaxSector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5687) /* Skip discard while reshape is happening */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5688) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5690) logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5691) last_sector = bio_end_sector(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5693) bi->bi_next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5695) stripe_sectors = conf->chunk_sectors *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5696) (conf->raid_disks - conf->max_degraded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5697) logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5698) stripe_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5699) sector_div(last_sector, stripe_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5701) logical_sector *= conf->chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5702) last_sector *= conf->chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5704) for (; logical_sector < last_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5705) logical_sector += RAID5_STRIPE_SECTORS(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5706) DEFINE_WAIT(w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5707) int d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5708) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5709) sh = raid5_get_active_stripe(conf, logical_sector, 0, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5710) prepare_to_wait(&conf->wait_for_overlap, &w,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5711) TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5712) set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5713) if (test_bit(STRIPE_SYNCING, &sh->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5714) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5715) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5716) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5718) clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5719) spin_lock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5720) for (d = 0; d < conf->raid_disks; d++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5721) if (d == sh->pd_idx || d == sh->qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5722) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5723) if (sh->dev[d].towrite || sh->dev[d].toread) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5724) set_bit(R5_Overlap, &sh->dev[d].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5725) spin_unlock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5726) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5727) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5728) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5730) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5731) set_bit(STRIPE_DISCARD, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5732) finish_wait(&conf->wait_for_overlap, &w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5733) sh->overwrite_disks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5734) for (d = 0; d < conf->raid_disks; d++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5735) if (d == sh->pd_idx || d == sh->qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5736) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5737) sh->dev[d].towrite = bi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5738) set_bit(R5_OVERWRITE, &sh->dev[d].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5739) bio_inc_remaining(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5740) md_write_inc(mddev, bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5741) sh->overwrite_disks++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5743) spin_unlock_irq(&sh->stripe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5744) if (conf->mddev->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5745) for (d = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5746) d < conf->raid_disks - conf->max_degraded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5747) d++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5748) md_bitmap_startwrite(mddev->bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5749) sh->sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5750) RAID5_STRIPE_SECTORS(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5751) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5752) sh->bm_seq = conf->seq_flush + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5753) set_bit(STRIPE_BIT_DELAY, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5756) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5757) clear_bit(STRIPE_DELAYED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5758) if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5759) atomic_inc(&conf->preread_active_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5760) release_stripe_plug(mddev, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5763) bio_endio(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5766) static bool raid5_make_request(struct mddev *mddev, struct bio * bi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5767) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5768) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5769) int dd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5770) sector_t new_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5771) sector_t logical_sector, last_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5772) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5773) const int rw = bio_data_dir(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5774) DEFINE_WAIT(w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5775) bool do_prepare;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5776) bool do_flush = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5778) if (unlikely(bi->bi_opf & REQ_PREFLUSH)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5779) int ret = log_handle_flush_request(conf, bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5781) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5782) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5783) if (ret == -ENODEV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5784) if (md_flush_request(mddev, bi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5785) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5787) /* ret == -EAGAIN, fallback */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5788) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5789) * if r5l_handle_flush_request() didn't clear REQ_PREFLUSH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5790) * we need to flush journal device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5791) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5792) do_flush = bi->bi_opf & REQ_PREFLUSH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5795) if (!md_write_start(mddev, bi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5796) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5797) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5798) * If array is degraded, better not do chunk aligned read because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5799) * later we might have to read it again in order to reconstruct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5800) * data on failed drives.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5801) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5802) if (rw == READ && mddev->degraded == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5803) mddev->reshape_position == MaxSector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5804) bi = chunk_aligned_read(mddev, bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5805) if (!bi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5806) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5809) if (unlikely(bio_op(bi) == REQ_OP_DISCARD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5810) make_discard_request(mddev, bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5811) md_write_end(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5812) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5815) logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5816) last_sector = bio_end_sector(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5817) bi->bi_next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5819) prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5820) for (; logical_sector < last_sector; logical_sector += RAID5_STRIPE_SECTORS(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5821) int previous;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5822) int seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5824) do_prepare = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5825) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5826) seq = read_seqcount_begin(&conf->gen_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5827) previous = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5828) if (do_prepare)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5829) prepare_to_wait(&conf->wait_for_overlap, &w,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5830) TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5831) if (unlikely(conf->reshape_progress != MaxSector)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5832) /* spinlock is needed as reshape_progress may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5833) * 64bit on a 32bit platform, and so it might be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5834) * possible to see a half-updated value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5835) * Of course reshape_progress could change after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5836) * the lock is dropped, so once we get a reference
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5837) * to the stripe that we think it is, we will have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5838) * to check again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5839) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5840) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5841) if (mddev->reshape_backwards
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5842) ? logical_sector < conf->reshape_progress
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5843) : logical_sector >= conf->reshape_progress) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5844) previous = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5845) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5846) if (mddev->reshape_backwards
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5847) ? logical_sector < conf->reshape_safe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5848) : logical_sector >= conf->reshape_safe) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5849) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5850) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5851) do_prepare = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5852) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5855) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5858) new_sector = raid5_compute_sector(conf, logical_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5859) previous,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5860) &dd_idx, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5861) pr_debug("raid456: raid5_make_request, sector %llu logical %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5862) (unsigned long long)new_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5863) (unsigned long long)logical_sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5865) sh = raid5_get_active_stripe(conf, new_sector, previous,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5866) (bi->bi_opf & REQ_RAHEAD), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5867) if (sh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5868) if (unlikely(previous)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5869) /* expansion might have moved on while waiting for a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5870) * stripe, so we must do the range check again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5871) * Expansion could still move past after this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5872) * test, but as we are holding a reference to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5873) * 'sh', we know that if that happens,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5874) * STRIPE_EXPANDING will get set and the expansion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5875) * won't proceed until we finish with the stripe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5876) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5877) int must_retry = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5878) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5879) if (mddev->reshape_backwards
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5880) ? logical_sector >= conf->reshape_progress
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5881) : logical_sector < conf->reshape_progress)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5882) /* mismatch, need to try again */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5883) must_retry = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5884) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5885) if (must_retry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5886) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5887) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5888) do_prepare = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5889) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5892) if (read_seqcount_retry(&conf->gen_lock, seq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5893) /* Might have got the wrong stripe_head
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5894) * by accident
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5896) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5897) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5900) if (test_bit(STRIPE_EXPANDING, &sh->state) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5901) !add_stripe_bio(sh, bi, dd_idx, rw, previous)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5902) /* Stripe is busy expanding or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5903) * add failed due to overlap. Flush everything
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5904) * and wait a while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5905) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5906) md_wakeup_thread(mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5907) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5908) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5909) do_prepare = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5910) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5912) if (do_flush) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5913) set_bit(STRIPE_R5C_PREFLUSH, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5914) /* we only need flush for one stripe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5915) do_flush = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5916) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5918) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5919) clear_bit(STRIPE_DELAYED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5920) if ((!sh->batch_head || sh == sh->batch_head) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5921) (bi->bi_opf & REQ_SYNC) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5922) !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5923) atomic_inc(&conf->preread_active_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5924) release_stripe_plug(mddev, sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5925) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5926) /* cannot get stripe for read-ahead, just give-up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5927) bi->bi_status = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5928) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5929) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5931) finish_wait(&conf->wait_for_overlap, &w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5933) if (rw == WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5934) md_write_end(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5935) bio_endio(bi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5936) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5939) static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5941) static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5942) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5943) /* reshaping is quite different to recovery/resync so it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5944) * handled quite separately ... here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5945) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5946) * On each call to sync_request, we gather one chunk worth of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5947) * destination stripes and flag them as expanding.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5948) * Then we find all the source stripes and request reads.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5949) * As the reads complete, handle_stripe will copy the data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5950) * into the destination stripe and release that stripe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5951) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5952) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5953) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5954) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5955) sector_t first_sector, last_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5956) int raid_disks = conf->previous_raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5957) int data_disks = raid_disks - conf->max_degraded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5958) int new_data_disks = conf->raid_disks - conf->max_degraded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5959) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5960) int dd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5961) sector_t writepos, readpos, safepos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5962) sector_t stripe_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5963) int reshape_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5964) struct list_head stripes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5965) sector_t retn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5967) if (sector_nr == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5968) /* If restarting in the middle, skip the initial sectors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5969) if (mddev->reshape_backwards &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5970) conf->reshape_progress < raid5_size(mddev, 0, 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5971) sector_nr = raid5_size(mddev, 0, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5972) - conf->reshape_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5973) } else if (mddev->reshape_backwards &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5974) conf->reshape_progress == MaxSector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5975) /* shouldn't happen, but just in case, finish up.*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5976) sector_nr = MaxSector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5977) } else if (!mddev->reshape_backwards &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5978) conf->reshape_progress > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5979) sector_nr = conf->reshape_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5980) sector_div(sector_nr, new_data_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5981) if (sector_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5982) mddev->curr_resync_completed = sector_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5983) sysfs_notify_dirent_safe(mddev->sysfs_completed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5984) *skipped = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5985) retn = sector_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5986) goto finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5987) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5990) /* We need to process a full chunk at a time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5991) * If old and new chunk sizes differ, we need to process the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5992) * largest of these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5993) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5995) reshape_sectors = max(conf->chunk_sectors, conf->prev_chunk_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5997) /* We update the metadata at least every 10 seconds, or when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5998) * the data about to be copied would over-write the source of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5999) * the data at the front of the range. i.e. one new_stripe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6000) * along from reshape_progress new_maps to after where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6001) * reshape_safe old_maps to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6002) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6003) writepos = conf->reshape_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6004) sector_div(writepos, new_data_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6005) readpos = conf->reshape_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6006) sector_div(readpos, data_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6007) safepos = conf->reshape_safe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6008) sector_div(safepos, data_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6009) if (mddev->reshape_backwards) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6010) BUG_ON(writepos < reshape_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6011) writepos -= reshape_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6012) readpos += reshape_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6013) safepos += reshape_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6014) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6015) writepos += reshape_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6016) /* readpos and safepos are worst-case calculations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6017) * A negative number is overly pessimistic, and causes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6018) * obvious problems for unsigned storage. So clip to 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6019) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6020) readpos -= min_t(sector_t, reshape_sectors, readpos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6021) safepos -= min_t(sector_t, reshape_sectors, safepos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6022) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6024) /* Having calculated the 'writepos' possibly use it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6025) * to set 'stripe_addr' which is where we will write to.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6026) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6027) if (mddev->reshape_backwards) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6028) BUG_ON(conf->reshape_progress == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6029) stripe_addr = writepos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6030) BUG_ON((mddev->dev_sectors &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6031) ~((sector_t)reshape_sectors - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6032) - reshape_sectors - stripe_addr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6033) != sector_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6034) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6035) BUG_ON(writepos != sector_nr + reshape_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6036) stripe_addr = sector_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6037) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6039) /* 'writepos' is the most advanced device address we might write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6040) * 'readpos' is the least advanced device address we might read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6041) * 'safepos' is the least address recorded in the metadata as having
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6042) * been reshaped.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6043) * If there is a min_offset_diff, these are adjusted either by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6044) * increasing the safepos/readpos if diff is negative, or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6045) * increasing writepos if diff is positive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6046) * If 'readpos' is then behind 'writepos', there is no way that we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6047) * ensure safety in the face of a crash - that must be done by userspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6048) * making a backup of the data. So in that case there is no particular
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6049) * rush to update metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6050) * Otherwise if 'safepos' is behind 'writepos', then we really need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6051) * update the metadata to advance 'safepos' to match 'readpos' so that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6052) * we can be safe in the event of a crash.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6053) * So we insist on updating metadata if safepos is behind writepos and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6054) * readpos is beyond writepos.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6055) * In any case, update the metadata every 10 seconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6056) * Maybe that number should be configurable, but I'm not sure it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6057) * worth it.... maybe it could be a multiple of safemode_delay???
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6058) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6059) if (conf->min_offset_diff < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6060) safepos += -conf->min_offset_diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6061) readpos += -conf->min_offset_diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6062) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6063) writepos += conf->min_offset_diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6065) if ((mddev->reshape_backwards
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6066) ? (safepos > writepos && readpos < writepos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6067) : (safepos < writepos && readpos > writepos)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6068) time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6069) /* Cannot proceed until we've updated the superblock... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6070) wait_event(conf->wait_for_overlap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6071) atomic_read(&conf->reshape_stripes)==0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6072) || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6073) if (atomic_read(&conf->reshape_stripes) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6074) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6075) mddev->reshape_position = conf->reshape_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6076) mddev->curr_resync_completed = sector_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6077) if (!mddev->reshape_backwards)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6078) /* Can update recovery_offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6079) rdev_for_each(rdev, mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6080) if (rdev->raid_disk >= 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6081) !test_bit(Journal, &rdev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6082) !test_bit(In_sync, &rdev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6083) rdev->recovery_offset < sector_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6084) rdev->recovery_offset = sector_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6086) conf->reshape_checkpoint = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6087) set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6088) md_wakeup_thread(mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6089) wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6090) test_bit(MD_RECOVERY_INTR, &mddev->recovery));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6091) if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6092) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6093) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6094) conf->reshape_safe = mddev->reshape_position;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6095) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6096) wake_up(&conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6097) sysfs_notify_dirent_safe(mddev->sysfs_completed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6098) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6100) INIT_LIST_HEAD(&stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6101) for (i = 0; i < reshape_sectors; i += RAID5_STRIPE_SECTORS(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6102) int j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6103) int skipped_disk = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6104) sh = raid5_get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6105) set_bit(STRIPE_EXPANDING, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6106) atomic_inc(&conf->reshape_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6107) /* If any of this stripe is beyond the end of the old
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6108) * array, then we need to zero those blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6109) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6110) for (j=sh->disks; j--;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6111) sector_t s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6112) if (j == sh->pd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6113) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6114) if (conf->level == 6 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6115) j == sh->qd_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6116) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6117) s = raid5_compute_blocknr(sh, j, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6118) if (s < raid5_size(mddev, 0, 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6119) skipped_disk = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6120) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6122) memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6123) set_bit(R5_Expanded, &sh->dev[j].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6124) set_bit(R5_UPTODATE, &sh->dev[j].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6126) if (!skipped_disk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6127) set_bit(STRIPE_EXPAND_READY, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6128) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6130) list_add(&sh->lru, &stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6132) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6133) if (mddev->reshape_backwards)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6134) conf->reshape_progress -= reshape_sectors * new_data_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6135) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6136) conf->reshape_progress += reshape_sectors * new_data_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6137) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6138) /* Ok, those stripe are ready. We can start scheduling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6139) * reads on the source stripes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6140) * The source stripes are determined by mapping the first and last
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6141) * block on the destination stripes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6142) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6143) first_sector =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6144) raid5_compute_sector(conf, stripe_addr*(new_data_disks),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6145) 1, &dd_idx, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6146) last_sector =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6147) raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6148) * new_data_disks - 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6149) 1, &dd_idx, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6150) if (last_sector >= mddev->dev_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6151) last_sector = mddev->dev_sectors - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6152) while (first_sector <= last_sector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6153) sh = raid5_get_active_stripe(conf, first_sector, 1, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6154) set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6155) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6156) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6157) first_sector += RAID5_STRIPE_SECTORS(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6159) /* Now that the sources are clearly marked, we can release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6160) * the destination stripes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6161) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6162) while (!list_empty(&stripes)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6163) sh = list_entry(stripes.next, struct stripe_head, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6164) list_del_init(&sh->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6165) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6167) /* If this takes us to the resync_max point where we have to pause,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6168) * then we need to write out the superblock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6169) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6170) sector_nr += reshape_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6171) retn = reshape_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6172) finish:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6173) if (mddev->curr_resync_completed > mddev->resync_max ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6174) (sector_nr - mddev->curr_resync_completed) * 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6175) >= mddev->resync_max - mddev->curr_resync_completed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6176) /* Cannot proceed until we've updated the superblock... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6177) wait_event(conf->wait_for_overlap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6178) atomic_read(&conf->reshape_stripes) == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6179) || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6180) if (atomic_read(&conf->reshape_stripes) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6181) goto ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6182) mddev->reshape_position = conf->reshape_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6183) mddev->curr_resync_completed = sector_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6184) if (!mddev->reshape_backwards)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6185) /* Can update recovery_offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6186) rdev_for_each(rdev, mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6187) if (rdev->raid_disk >= 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6188) !test_bit(Journal, &rdev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6189) !test_bit(In_sync, &rdev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6190) rdev->recovery_offset < sector_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6191) rdev->recovery_offset = sector_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6192) conf->reshape_checkpoint = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6193) set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6194) md_wakeup_thread(mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6195) wait_event(mddev->sb_wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6196) !test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6197) || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6198) if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6199) goto ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6200) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6201) conf->reshape_safe = mddev->reshape_position;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6202) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6203) wake_up(&conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6204) sysfs_notify_dirent_safe(mddev->sysfs_completed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6206) ret:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6207) return retn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6210) static inline sector_t raid5_sync_request(struct mddev *mddev, sector_t sector_nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6211) int *skipped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6212) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6213) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6214) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6215) sector_t max_sector = mddev->dev_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6216) sector_t sync_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6217) int still_degraded = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6218) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6220) if (sector_nr >= max_sector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6221) /* just being told to finish up .. nothing much to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6223) if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6224) end_reshape(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6225) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6228) if (mddev->curr_resync < max_sector) /* aborted */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6229) md_bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6230) &sync_blocks, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6231) else /* completed sync */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6232) conf->fullsync = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6233) md_bitmap_close_sync(mddev->bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6235) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6238) /* Allow raid5_quiesce to complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6239) wait_event(conf->wait_for_overlap, conf->quiesce != 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6241) if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6242) return reshape_request(mddev, sector_nr, skipped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6244) /* No need to check resync_max as we never do more than one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6245) * stripe, and as resync_max will always be on a chunk boundary,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6246) * if the check in md_do_sync didn't fire, there is no chance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6247) * of overstepping resync_max here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6248) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6250) /* if there is too many failed drives and we are trying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6251) * to resync, then assert that we are finished, because there is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6252) * nothing we can do.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6253) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6254) if (mddev->degraded >= conf->max_degraded &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6255) test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6256) sector_t rv = mddev->dev_sectors - sector_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6257) *skipped = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6258) return rv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6260) if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6261) !conf->fullsync &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6262) !md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6263) sync_blocks >= RAID5_STRIPE_SECTORS(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6264) /* we can skip this block, and probably more */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6265) do_div(sync_blocks, RAID5_STRIPE_SECTORS(conf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6266) *skipped = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6267) /* keep things rounded to whole stripes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6268) return sync_blocks * RAID5_STRIPE_SECTORS(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6271) md_bitmap_cond_end_sync(mddev->bitmap, sector_nr, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6273) sh = raid5_get_active_stripe(conf, sector_nr, 0, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6274) if (sh == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6275) sh = raid5_get_active_stripe(conf, sector_nr, 0, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6276) /* make sure we don't swamp the stripe cache if someone else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6277) * is trying to get access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6278) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6279) schedule_timeout_uninterruptible(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6281) /* Need to check if array will still be degraded after recovery/resync
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6282) * Note in case of > 1 drive failures it's possible we're rebuilding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6283) * one drive while leaving another faulty drive in array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6284) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6285) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6286) for (i = 0; i < conf->raid_disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6287) struct md_rdev *rdev = READ_ONCE(conf->disks[i].rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6289) if (rdev == NULL || test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6290) still_degraded = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6292) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6294) md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6296) set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6297) set_bit(STRIPE_HANDLE, &sh->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6299) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6301) return RAID5_STRIPE_SECTORS(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6304) static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6305) unsigned int offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6307) /* We may not be able to submit a whole bio at once as there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6308) * may not be enough stripe_heads available.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6309) * We cannot pre-allocate enough stripe_heads as we may need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6310) * more than exist in the cache (if we allow ever large chunks).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6311) * So we do one stripe head at a time and record in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6312) * ->bi_hw_segments how many have been done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6313) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6314) * We *know* that this entire raid_bio is in one chunk, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6315) * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6316) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6317) struct stripe_head *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6318) int dd_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6319) sector_t sector, logical_sector, last_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6320) int scnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6321) int handled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6323) logical_sector = raid_bio->bi_iter.bi_sector &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6324) ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6325) sector = raid5_compute_sector(conf, logical_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6326) 0, &dd_idx, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6327) last_sector = bio_end_sector(raid_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6329) for (; logical_sector < last_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6330) logical_sector += RAID5_STRIPE_SECTORS(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6331) sector += RAID5_STRIPE_SECTORS(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6332) scnt++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6334) if (scnt < offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6335) /* already done this stripe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6336) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6338) sh = raid5_get_active_stripe(conf, sector, 0, 1, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6340) if (!sh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6341) /* failed to get a stripe - must wait */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6342) conf->retry_read_aligned = raid_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6343) conf->retry_read_offset = scnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6344) return handled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6347) if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6348) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6349) conf->retry_read_aligned = raid_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6350) conf->retry_read_offset = scnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6351) return handled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6354) set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6355) handle_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6356) raid5_release_stripe(sh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6357) handled++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6360) bio_endio(raid_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6362) if (atomic_dec_and_test(&conf->active_aligned_reads))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6363) wake_up(&conf->wait_for_quiescent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6364) return handled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6367) static int handle_active_stripes(struct r5conf *conf, int group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6368) struct r5worker *worker,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6369) struct list_head *temp_inactive_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6370) __releases(&conf->device_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6371) __acquires(&conf->device_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6373) struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6374) int i, batch_size = 0, hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6375) bool release_inactive = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6377) while (batch_size < MAX_STRIPE_BATCH &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6378) (sh = __get_priority_stripe(conf, group)) != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6379) batch[batch_size++] = sh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6381) if (batch_size == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6382) for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6383) if (!list_empty(temp_inactive_list + i))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6384) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6385) if (i == NR_STRIPE_HASH_LOCKS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6386) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6387) log_flush_stripe_to_raid(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6388) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6389) return batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6391) release_inactive = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6393) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6395) release_inactive_stripe_list(conf, temp_inactive_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6396) NR_STRIPE_HASH_LOCKS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6398) r5l_flush_stripe_to_raid(conf->log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6399) if (release_inactive) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6400) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6401) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6404) for (i = 0; i < batch_size; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6405) handle_stripe(batch[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6406) log_write_stripe_run(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6408) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6410) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6411) for (i = 0; i < batch_size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6412) hash = batch[i]->hash_lock_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6413) __release_stripe(conf, batch[i], &temp_inactive_list[hash]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6415) return batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6418) static void raid5_do_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6420) struct r5worker *worker = container_of(work, struct r5worker, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6421) struct r5worker_group *group = worker->group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6422) struct r5conf *conf = group->conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6423) struct mddev *mddev = conf->mddev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6424) int group_id = group - conf->worker_groups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6425) int handled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6426) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6428) pr_debug("+++ raid5worker active\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6430) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6431) handled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6432) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6433) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6434) int batch_size, released;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6436) released = release_stripe_list(conf, worker->temp_inactive_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6438) batch_size = handle_active_stripes(conf, group_id, worker,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6439) worker->temp_inactive_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6440) worker->working = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6441) if (!batch_size && !released)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6442) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6443) handled += batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6444) wait_event_lock_irq(mddev->sb_wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6445) !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6446) conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6448) pr_debug("%d stripes handled\n", handled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6450) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6452) flush_deferred_bios(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6454) r5l_flush_stripe_to_raid(conf->log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6456) async_tx_issue_pending_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6457) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6459) pr_debug("--- raid5worker inactive\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6462) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6463) * This is our raid5 kernel thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6464) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6465) * We scan the hash table for stripes which can be handled now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6466) * During the scan, completed stripes are saved for us by the interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6467) * handler, so that they will not have to wait for our next wakeup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6468) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6469) static void raid5d(struct md_thread *thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6470) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6471) struct mddev *mddev = thread->mddev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6472) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6473) int handled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6474) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6476) pr_debug("+++ raid5d active\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6478) md_check_recovery(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6480) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6481) handled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6482) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6483) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6484) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6485) int batch_size, released;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6486) unsigned int offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6488) released = release_stripe_list(conf, conf->temp_inactive_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6489) if (released)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6490) clear_bit(R5_DID_ALLOC, &conf->cache_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6492) if (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6493) !list_empty(&conf->bitmap_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6494) /* Now is a good time to flush some bitmap updates */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6495) conf->seq_flush++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6496) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6497) md_bitmap_unplug(mddev->bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6498) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6499) conf->seq_write = conf->seq_flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6500) activate_bit_delay(conf, conf->temp_inactive_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6502) raid5_activate_delayed(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6504) while ((bio = remove_bio_from_retry(conf, &offset))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6505) int ok;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6506) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6507) ok = retry_aligned_read(conf, bio, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6508) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6509) if (!ok)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6510) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6511) handled++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6514) batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6515) conf->temp_inactive_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6516) if (!batch_size && !released)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6517) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6518) handled += batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6520) if (mddev->sb_flags & ~(1 << MD_SB_CHANGE_PENDING)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6521) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6522) md_check_recovery(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6523) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6526) pr_debug("%d stripes handled\n", handled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6528) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6529) if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6530) mutex_trylock(&conf->cache_size_mutex)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6531) grow_one_stripe(conf, __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6532) /* Set flag even if allocation failed. This helps
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6533) * slow down allocation requests when mem is short
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6534) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6535) set_bit(R5_DID_ALLOC, &conf->cache_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6536) mutex_unlock(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6539) flush_deferred_bios(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6541) r5l_flush_stripe_to_raid(conf->log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6543) async_tx_issue_pending_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6544) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6546) pr_debug("--- raid5d inactive\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6549) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6550) raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6552) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6553) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6554) spin_lock(&mddev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6555) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6556) if (conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6557) ret = sprintf(page, "%d\n", conf->min_nr_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6558) spin_unlock(&mddev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6559) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6562) int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6563) raid5_set_cache_size(struct mddev *mddev, int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6565) int result = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6566) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6568) if (size <= 16 || size > 32768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6569) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6571) conf->min_nr_stripes = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6572) mutex_lock(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6573) while (size < conf->max_nr_stripes &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6574) drop_one_stripe(conf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6575) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6576) mutex_unlock(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6578) md_allow_write(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6580) mutex_lock(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6581) while (size > conf->max_nr_stripes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6582) if (!grow_one_stripe(conf, GFP_KERNEL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6583) conf->min_nr_stripes = conf->max_nr_stripes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6584) result = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6585) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6587) mutex_unlock(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6589) return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6591) EXPORT_SYMBOL(raid5_set_cache_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6593) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6594) raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6595) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6596) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6597) unsigned long new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6598) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6600) if (len >= PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6601) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6602) if (kstrtoul(page, 10, &new))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6603) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6604) err = mddev_lock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6605) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6606) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6607) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6608) if (!conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6609) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6610) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6611) err = raid5_set_cache_size(mddev, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6612) mddev_unlock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6614) return err ?: len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6617) static struct md_sysfs_entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6618) raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6619) raid5_show_stripe_cache_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6620) raid5_store_stripe_cache_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6622) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6623) raid5_show_rmw_level(struct mddev *mddev, char *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6625) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6626) if (conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6627) return sprintf(page, "%d\n", conf->rmw_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6628) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6629) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6632) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6633) raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6634) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6635) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6636) unsigned long new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6638) if (!conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6639) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6641) if (len >= PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6642) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6644) if (kstrtoul(page, 10, &new))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6645) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6647) if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6648) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6650) if (new != PARITY_DISABLE_RMW &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6651) new != PARITY_ENABLE_RMW &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6652) new != PARITY_PREFER_RMW)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6653) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6655) conf->rmw_level = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6656) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6659) static struct md_sysfs_entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6660) raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6661) raid5_show_rmw_level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6662) raid5_store_rmw_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6664) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6665) raid5_show_stripe_size(struct mddev *mddev, char *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6666) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6667) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6668) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6670) spin_lock(&mddev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6671) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6672) if (conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6673) ret = sprintf(page, "%lu\n", RAID5_STRIPE_SIZE(conf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6674) spin_unlock(&mddev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6675) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6676) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6678) #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6679) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6680) raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6681) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6682) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6683) unsigned long new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6684) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6685) int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6687) if (len >= PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6688) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6689) if (kstrtoul(page, 10, &new))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6690) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6692) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6693) * The value should not be bigger than PAGE_SIZE. It requires to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6694) * be multiple of DEFAULT_STRIPE_SIZE and the value should be power
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6695) * of two.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6696) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6697) if (new % DEFAULT_STRIPE_SIZE != 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6698) new > PAGE_SIZE || new == 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6699) new != roundup_pow_of_two(new))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6700) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6702) err = mddev_lock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6703) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6704) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6706) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6707) if (!conf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6708) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6709) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6712) if (new == conf->stripe_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6713) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6715) pr_debug("md/raid: change stripe_size from %lu to %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6716) conf->stripe_size, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6718) if (mddev->sync_thread ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6719) test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6720) mddev->reshape_position != MaxSector ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6721) mddev->sysfs_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6722) err = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6723) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6724) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6726) mddev_suspend(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6727) mutex_lock(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6728) size = conf->max_nr_stripes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6730) shrink_stripes(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6732) conf->stripe_size = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6733) conf->stripe_shift = ilog2(new) - 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6734) conf->stripe_sectors = new >> 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6735) if (grow_stripes(conf, size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6736) pr_warn("md/raid:%s: couldn't allocate buffers\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6737) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6738) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6740) mutex_unlock(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6741) mddev_resume(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6743) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6744) mddev_unlock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6745) return err ?: len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6748) static struct md_sysfs_entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6749) raid5_stripe_size = __ATTR(stripe_size, 0644,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6750) raid5_show_stripe_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6751) raid5_store_stripe_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6752) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6753) static struct md_sysfs_entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6754) raid5_stripe_size = __ATTR(stripe_size, 0444,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6755) raid5_show_stripe_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6756) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6757) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6759) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6760) raid5_show_preread_threshold(struct mddev *mddev, char *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6761) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6762) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6763) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6764) spin_lock(&mddev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6765) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6766) if (conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6767) ret = sprintf(page, "%d\n", conf->bypass_threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6768) spin_unlock(&mddev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6769) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6772) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6773) raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6774) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6775) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6776) unsigned long new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6777) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6779) if (len >= PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6780) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6781) if (kstrtoul(page, 10, &new))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6782) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6784) err = mddev_lock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6785) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6786) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6787) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6788) if (!conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6789) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6790) else if (new > conf->min_nr_stripes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6791) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6792) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6793) conf->bypass_threshold = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6794) mddev_unlock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6795) return err ?: len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6798) static struct md_sysfs_entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6799) raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6800) S_IRUGO | S_IWUSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6801) raid5_show_preread_threshold,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6802) raid5_store_preread_threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6804) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6805) raid5_show_skip_copy(struct mddev *mddev, char *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6806) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6807) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6808) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6809) spin_lock(&mddev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6810) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6811) if (conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6812) ret = sprintf(page, "%d\n", conf->skip_copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6813) spin_unlock(&mddev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6814) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6815) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6817) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6818) raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6819) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6820) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6821) unsigned long new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6822) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6824) if (len >= PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6825) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6826) if (kstrtoul(page, 10, &new))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6827) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6828) new = !!new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6830) err = mddev_lock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6831) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6832) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6833) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6834) if (!conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6835) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6836) else if (new != conf->skip_copy) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6837) struct request_queue *q = mddev->queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6839) mddev_suspend(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6840) conf->skip_copy = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6841) if (new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6842) blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6843) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6844) blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6845) mddev_resume(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6847) mddev_unlock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6848) return err ?: len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6849) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6851) static struct md_sysfs_entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6852) raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6853) raid5_show_skip_copy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6854) raid5_store_skip_copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6856) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6857) stripe_cache_active_show(struct mddev *mddev, char *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6858) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6859) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6860) if (conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6861) return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6862) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6863) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6866) static struct md_sysfs_entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6867) raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6869) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6870) raid5_show_group_thread_cnt(struct mddev *mddev, char *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6871) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6872) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6873) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6874) spin_lock(&mddev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6875) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6876) if (conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6877) ret = sprintf(page, "%d\n", conf->worker_cnt_per_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6878) spin_unlock(&mddev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6879) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6882) static int alloc_thread_groups(struct r5conf *conf, int cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6883) int *group_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6884) struct r5worker_group **worker_groups);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6885) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6886) raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6887) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6888) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6889) unsigned int new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6890) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6891) struct r5worker_group *new_groups, *old_groups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6892) int group_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6894) if (len >= PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6895) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6896) if (kstrtouint(page, 10, &new))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6897) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6898) /* 8192 should be big enough */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6899) if (new > 8192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6900) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6902) err = mddev_lock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6903) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6904) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6905) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6906) if (!conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6907) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6908) else if (new != conf->worker_cnt_per_group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6909) mddev_suspend(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6911) old_groups = conf->worker_groups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6912) if (old_groups)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6913) flush_workqueue(raid5_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6915) err = alloc_thread_groups(conf, new, &group_cnt, &new_groups);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6916) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6917) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6918) conf->group_cnt = group_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6919) conf->worker_cnt_per_group = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6920) conf->worker_groups = new_groups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6921) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6923) if (old_groups)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6924) kfree(old_groups[0].workers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6925) kfree(old_groups);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6926) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6927) mddev_resume(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6928) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6929) mddev_unlock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6931) return err ?: len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6932) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6934) static struct md_sysfs_entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6935) raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6936) raid5_show_group_thread_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6937) raid5_store_group_thread_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6939) static struct attribute *raid5_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6940) &raid5_stripecache_size.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6941) &raid5_stripecache_active.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6942) &raid5_preread_bypass_threshold.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6943) &raid5_group_thread_cnt.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6944) &raid5_skip_copy.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6945) &raid5_rmw_level.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6946) &raid5_stripe_size.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6947) &r5c_journal_mode.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6948) &ppl_write_hint.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6949) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6950) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6951) static struct attribute_group raid5_attrs_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6952) .name = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6953) .attrs = raid5_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6954) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6956) static int alloc_thread_groups(struct r5conf *conf, int cnt, int *group_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6957) struct r5worker_group **worker_groups)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6958) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6959) int i, j, k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6960) ssize_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6961) struct r5worker *workers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6963) if (cnt == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6964) *group_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6965) *worker_groups = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6966) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6967) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6968) *group_cnt = num_possible_nodes();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6969) size = sizeof(struct r5worker) * cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6970) workers = kcalloc(size, *group_cnt, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6971) *worker_groups = kcalloc(*group_cnt, sizeof(struct r5worker_group),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6972) GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6973) if (!*worker_groups || !workers) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6974) kfree(workers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6975) kfree(*worker_groups);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6976) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6977) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6979) for (i = 0; i < *group_cnt; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6980) struct r5worker_group *group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6982) group = &(*worker_groups)[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6983) INIT_LIST_HEAD(&group->handle_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6984) INIT_LIST_HEAD(&group->loprio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6985) group->conf = conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6986) group->workers = workers + i * cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6988) for (j = 0; j < cnt; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6989) struct r5worker *worker = group->workers + j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6990) worker->group = group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6991) INIT_WORK(&worker->work, raid5_do_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6993) for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6994) INIT_LIST_HEAD(worker->temp_inactive_list + k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6998) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7001) static void free_thread_groups(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7002) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7003) if (conf->worker_groups)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7004) kfree(conf->worker_groups[0].workers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7005) kfree(conf->worker_groups);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7006) conf->worker_groups = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7007) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7009) static sector_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7010) raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7011) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7012) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7014) if (!sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7015) sectors = mddev->dev_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7016) if (!raid_disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7017) /* size is defined by the smallest of previous and new size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7018) raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7020) sectors &= ~((sector_t)conf->chunk_sectors - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7021) sectors &= ~((sector_t)conf->prev_chunk_sectors - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7022) return sectors * (raid_disks - conf->max_degraded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7023) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7025) static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7026) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7027) safe_put_page(percpu->spare_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7028) percpu->spare_page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7029) kvfree(percpu->scribble);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7030) percpu->scribble = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7031) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7032)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7033) static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7034) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7035) if (conf->level == 6 && !percpu->spare_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7036) percpu->spare_page = alloc_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7037) if (!percpu->spare_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7038) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7039) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7041) if (scribble_alloc(percpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7042) max(conf->raid_disks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7043) conf->previous_raid_disks),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7044) max(conf->chunk_sectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7045) conf->prev_chunk_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7046) / RAID5_STRIPE_SECTORS(conf))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7047) free_scratch_buffer(conf, percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7048) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7049) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7051) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7054) static int raid456_cpu_dead(unsigned int cpu, struct hlist_node *node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7055) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7056) struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7058) free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7059) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7062) static void raid5_free_percpu(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7063) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7064) if (!conf->percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7065) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7067) cpuhp_state_remove_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7068) free_percpu(conf->percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7069) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7071) static void free_conf(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7072) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7073) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7075) log_exit(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7077) unregister_shrinker(&conf->shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7078) free_thread_groups(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7079) shrink_stripes(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7080) raid5_free_percpu(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7081) for (i = 0; i < conf->pool_size; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7082) if (conf->disks[i].extra_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7083) put_page(conf->disks[i].extra_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7084) kfree(conf->disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7085) bioset_exit(&conf->bio_split);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7086) kfree(conf->stripe_hashtbl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7087) kfree(conf->pending_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7088) kfree(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7091) static int raid456_cpu_up_prepare(unsigned int cpu, struct hlist_node *node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7092) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7093) struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7094) struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7096) if (alloc_scratch_buffer(conf, percpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7097) pr_warn("%s: failed memory allocation for cpu%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7098) __func__, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7099) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7101) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7104) static int raid5_alloc_percpu(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7106) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7108) conf->percpu = alloc_percpu(struct raid5_percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7109) if (!conf->percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7110) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7112) err = cpuhp_state_add_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7113) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7114) conf->scribble_disks = max(conf->raid_disks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7115) conf->previous_raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7116) conf->scribble_sectors = max(conf->chunk_sectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7117) conf->prev_chunk_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7119) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7122) static unsigned long raid5_cache_scan(struct shrinker *shrink,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7123) struct shrink_control *sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7125) struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7126) unsigned long ret = SHRINK_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7128) if (mutex_trylock(&conf->cache_size_mutex)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7129) ret= 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7130) while (ret < sc->nr_to_scan &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7131) conf->max_nr_stripes > conf->min_nr_stripes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7132) if (drop_one_stripe(conf) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7133) ret = SHRINK_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7134) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7136) ret++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7138) mutex_unlock(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7140) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7143) static unsigned long raid5_cache_count(struct shrinker *shrink,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7144) struct shrink_control *sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7146) struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7148) if (conf->max_nr_stripes < conf->min_nr_stripes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7149) /* unlikely, but not impossible */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7150) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7151) return conf->max_nr_stripes - conf->min_nr_stripes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7154) static struct r5conf *setup_conf(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7156) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7157) int raid_disk, memory, max_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7158) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7159) struct disk_info *disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7160) char pers_name[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7161) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7162) int group_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7163) struct r5worker_group *new_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7164) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7166) if (mddev->new_level != 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7167) && mddev->new_level != 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7168) && mddev->new_level != 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7169) pr_warn("md/raid:%s: raid level not set to 4/5/6 (%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7170) mdname(mddev), mddev->new_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7171) return ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7173) if ((mddev->new_level == 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7174) && !algorithm_valid_raid5(mddev->new_layout)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7175) (mddev->new_level == 6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7176) && !algorithm_valid_raid6(mddev->new_layout))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7177) pr_warn("md/raid:%s: layout %d not supported\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7178) mdname(mddev), mddev->new_layout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7179) return ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7181) if (mddev->new_level == 6 && mddev->raid_disks < 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7182) pr_warn("md/raid:%s: not enough configured devices (%d, minimum 4)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7183) mdname(mddev), mddev->raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7184) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7187) if (!mddev->new_chunk_sectors ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7188) (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7189) !is_power_of_2(mddev->new_chunk_sectors)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7190) pr_warn("md/raid:%s: invalid chunk size %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7191) mdname(mddev), mddev->new_chunk_sectors << 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7192) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7195) conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7196) if (conf == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7197) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7199) #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7200) conf->stripe_size = DEFAULT_STRIPE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7201) conf->stripe_shift = ilog2(DEFAULT_STRIPE_SIZE) - 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7202) conf->stripe_sectors = DEFAULT_STRIPE_SIZE >> 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7203) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7204) INIT_LIST_HEAD(&conf->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7205) INIT_LIST_HEAD(&conf->pending_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7206) conf->pending_data = kcalloc(PENDING_IO_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7207) sizeof(struct r5pending_data),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7208) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7209) if (!conf->pending_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7210) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7211) for (i = 0; i < PENDING_IO_MAX; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7212) list_add(&conf->pending_data[i].sibling, &conf->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7213) /* Don't enable multi-threading by default*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7214) if (!alloc_thread_groups(conf, 0, &group_cnt, &new_group)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7215) conf->group_cnt = group_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7216) conf->worker_cnt_per_group = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7217) conf->worker_groups = new_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7218) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7219) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7220) spin_lock_init(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7221) seqcount_spinlock_init(&conf->gen_lock, &conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7222) mutex_init(&conf->cache_size_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7223) init_waitqueue_head(&conf->wait_for_quiescent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7224) init_waitqueue_head(&conf->wait_for_stripe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7225) init_waitqueue_head(&conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7226) INIT_LIST_HEAD(&conf->handle_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7227) INIT_LIST_HEAD(&conf->loprio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7228) INIT_LIST_HEAD(&conf->hold_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7229) INIT_LIST_HEAD(&conf->delayed_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7230) INIT_LIST_HEAD(&conf->bitmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7231) init_llist_head(&conf->released_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7232) atomic_set(&conf->active_stripes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7233) atomic_set(&conf->preread_active_stripes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7234) atomic_set(&conf->active_aligned_reads, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7235) spin_lock_init(&conf->pending_bios_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7236) conf->batch_bio_dispatch = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7237) rdev_for_each(rdev, mddev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7238) if (test_bit(Journal, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7239) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7240) if (blk_queue_nonrot(bdev_get_queue(rdev->bdev))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7241) conf->batch_bio_dispatch = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7242) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7246) conf->bypass_threshold = BYPASS_THRESHOLD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7247) conf->recovery_disabled = mddev->recovery_disabled - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7249) conf->raid_disks = mddev->raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7250) if (mddev->reshape_position == MaxSector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7251) conf->previous_raid_disks = mddev->raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7252) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7253) conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7254) max_disks = max(conf->raid_disks, conf->previous_raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7256) conf->disks = kcalloc(max_disks, sizeof(struct disk_info),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7257) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7259) if (!conf->disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7260) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7262) for (i = 0; i < max_disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7263) conf->disks[i].extra_page = alloc_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7264) if (!conf->disks[i].extra_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7265) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7268) ret = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7269) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7270) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7271) conf->mddev = mddev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7273) if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7274) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7276) /* We init hash_locks[0] separately to that it can be used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7277) * as the reference lock in the spin_lock_nest_lock() call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7278) * in lock_all_device_hash_locks_irq in order to convince
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7279) * lockdep that we know what we are doing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7280) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7281) spin_lock_init(conf->hash_locks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7282) for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7283) spin_lock_init(conf->hash_locks + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7285) for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7286) INIT_LIST_HEAD(conf->inactive_list + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7288) for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7289) INIT_LIST_HEAD(conf->temp_inactive_list + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7291) atomic_set(&conf->r5c_cached_full_stripes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7292) INIT_LIST_HEAD(&conf->r5c_full_stripe_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7293) atomic_set(&conf->r5c_cached_partial_stripes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7294) INIT_LIST_HEAD(&conf->r5c_partial_stripe_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7295) atomic_set(&conf->r5c_flushing_full_stripes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7296) atomic_set(&conf->r5c_flushing_partial_stripes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7298) conf->level = mddev->new_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7299) conf->chunk_sectors = mddev->new_chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7300) if (raid5_alloc_percpu(conf) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7301) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7303) pr_debug("raid456: run(%s) called.\n", mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7305) rdev_for_each(rdev, mddev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7306) raid_disk = rdev->raid_disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7307) if (raid_disk >= max_disks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7308) || raid_disk < 0 || test_bit(Journal, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7309) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7310) disk = conf->disks + raid_disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7312) if (test_bit(Replacement, &rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7313) if (disk->replacement)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7314) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7315) disk->replacement = rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7316) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7317) if (disk->rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7318) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7319) disk->rdev = rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7322) if (test_bit(In_sync, &rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7323) char b[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7324) pr_info("md/raid:%s: device %s operational as raid disk %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7325) mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7326) } else if (rdev->saved_raid_disk != raid_disk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7327) /* Cannot rely on bitmap to complete recovery */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7328) conf->fullsync = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7331) conf->level = mddev->new_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7332) if (conf->level == 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7333) conf->max_degraded = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7334) if (raid6_call.xor_syndrome)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7335) conf->rmw_level = PARITY_ENABLE_RMW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7336) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7337) conf->rmw_level = PARITY_DISABLE_RMW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7338) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7339) conf->max_degraded = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7340) conf->rmw_level = PARITY_ENABLE_RMW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7342) conf->algorithm = mddev->new_layout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7343) conf->reshape_progress = mddev->reshape_position;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7344) if (conf->reshape_progress != MaxSector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7345) conf->prev_chunk_sectors = mddev->chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7346) conf->prev_algo = mddev->layout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7347) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7348) conf->prev_chunk_sectors = conf->chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7349) conf->prev_algo = conf->algorithm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7352) conf->min_nr_stripes = NR_STRIPES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7353) if (mddev->reshape_position != MaxSector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7354) int stripes = max_t(int,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7355) ((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7356) ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7357) conf->min_nr_stripes = max(NR_STRIPES, stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7358) if (conf->min_nr_stripes != NR_STRIPES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7359) pr_info("md/raid:%s: force stripe size %d for reshape\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7360) mdname(mddev), conf->min_nr_stripes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7362) memory = conf->min_nr_stripes * (sizeof(struct stripe_head) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7363) max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7364) atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7365) if (grow_stripes(conf, conf->min_nr_stripes)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7366) pr_warn("md/raid:%s: couldn't allocate %dkB for buffers\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7367) mdname(mddev), memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7368) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7369) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7370) pr_debug("md/raid:%s: allocated %dkB\n", mdname(mddev), memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7371) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7372) * Losing a stripe head costs more than the time to refill it,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7373) * it reduces the queue depth and so can hurt throughput.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7374) * So set it rather large, scaled by number of devices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7375) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7376) conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7377) conf->shrinker.scan_objects = raid5_cache_scan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7378) conf->shrinker.count_objects = raid5_cache_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7379) conf->shrinker.batch = 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7380) conf->shrinker.flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7381) if (register_shrinker(&conf->shrinker)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7382) pr_warn("md/raid:%s: couldn't register shrinker.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7383) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7384) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7387) sprintf(pers_name, "raid%d", mddev->new_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7388) conf->thread = md_register_thread(raid5d, mddev, pers_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7389) if (!conf->thread) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7390) pr_warn("md/raid:%s: couldn't allocate thread.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7391) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7392) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7395) return conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7397) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7398) if (conf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7399) free_conf(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7400) return ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7401) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7402) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7405) static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7407) switch (algo) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7408) case ALGORITHM_PARITY_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7409) if (raid_disk < max_degraded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7410) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7411) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7412) case ALGORITHM_PARITY_N:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7413) if (raid_disk >= raid_disks - max_degraded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7414) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7415) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7416) case ALGORITHM_PARITY_0_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7417) if (raid_disk == 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7418) raid_disk == raid_disks - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7419) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7420) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7421) case ALGORITHM_LEFT_ASYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7422) case ALGORITHM_RIGHT_ASYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7423) case ALGORITHM_LEFT_SYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7424) case ALGORITHM_RIGHT_SYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7425) if (raid_disk == raid_disks - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7426) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7428) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7431) static void raid5_set_io_opt(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7432) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7433) blk_queue_io_opt(conf->mddev->queue, (conf->chunk_sectors << 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7434) (conf->raid_disks - conf->max_degraded));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7437) static int raid5_run(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7438) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7439) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7440) int working_disks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7441) int dirty_parity_disks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7442) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7443) struct md_rdev *journal_dev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7444) sector_t reshape_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7445) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7446) long long min_offset_diff = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7447) int first = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7449) if (mddev_init_writes_pending(mddev) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7450) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7452) if (mddev->recovery_cp != MaxSector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7453) pr_notice("md/raid:%s: not clean -- starting background reconstruction\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7454) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7456) rdev_for_each(rdev, mddev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7457) long long diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7459) if (test_bit(Journal, &rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7460) journal_dev = rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7461) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7463) if (rdev->raid_disk < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7464) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7465) diff = (rdev->new_data_offset - rdev->data_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7466) if (first) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7467) min_offset_diff = diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7468) first = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7469) } else if (mddev->reshape_backwards &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7470) diff < min_offset_diff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7471) min_offset_diff = diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7472) else if (!mddev->reshape_backwards &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7473) diff > min_offset_diff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7474) min_offset_diff = diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7477) if ((test_bit(MD_HAS_JOURNAL, &mddev->flags) || journal_dev) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7478) (mddev->bitmap_info.offset || mddev->bitmap_info.file)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7479) pr_notice("md/raid:%s: array cannot have both journal and bitmap\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7480) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7481) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7484) if (mddev->reshape_position != MaxSector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7485) /* Check that we can continue the reshape.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7486) * Difficulties arise if the stripe we would write to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7487) * next is at or after the stripe we would read from next.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7488) * For a reshape that changes the number of devices, this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7489) * is only possible for a very short time, and mdadm makes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7490) * sure that time appears to have past before assembling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7491) * the array. So we fail if that time hasn't passed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7492) * For a reshape that keeps the number of devices the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7493) * mdadm must be monitoring the reshape can keeping the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7494) * critical areas read-only and backed up. It will start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7495) * the array in read-only mode, so we check for that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7496) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7497) sector_t here_new, here_old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7498) int old_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7499) int max_degraded = (mddev->level == 6 ? 2 : 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7500) int chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7501) int new_data_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7503) if (journal_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7504) pr_warn("md/raid:%s: don't support reshape with journal - aborting.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7505) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7506) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7509) if (mddev->new_level != mddev->level) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7510) pr_warn("md/raid:%s: unsupported reshape required - aborting.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7511) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7512) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7514) old_disks = mddev->raid_disks - mddev->delta_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7515) /* reshape_position must be on a new-stripe boundary, and one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7516) * further up in new geometry must map after here in old
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7517) * geometry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7518) * If the chunk sizes are different, then as we perform reshape
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7519) * in units of the largest of the two, reshape_position needs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7520) * be a multiple of the largest chunk size times new data disks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7521) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7522) here_new = mddev->reshape_position;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7523) chunk_sectors = max(mddev->chunk_sectors, mddev->new_chunk_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7524) new_data_disks = mddev->raid_disks - max_degraded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7525) if (sector_div(here_new, chunk_sectors * new_data_disks)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7526) pr_warn("md/raid:%s: reshape_position not on a stripe boundary\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7527) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7528) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7530) reshape_offset = here_new * chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7531) /* here_new is the stripe we will write to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7532) here_old = mddev->reshape_position;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7533) sector_div(here_old, chunk_sectors * (old_disks-max_degraded));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7534) /* here_old is the first stripe that we might need to read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7535) * from */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7536) if (mddev->delta_disks == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7537) /* We cannot be sure it is safe to start an in-place
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7538) * reshape. It is only safe if user-space is monitoring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7539) * and taking constant backups.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7540) * mdadm always starts a situation like this in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7541) * readonly mode so it can take control before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7542) * allowing any writes. So just check for that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7543) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7544) if (abs(min_offset_diff) >= mddev->chunk_sectors &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7545) abs(min_offset_diff) >= mddev->new_chunk_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7546) /* not really in-place - so OK */;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7547) else if (mddev->ro == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7548) pr_warn("md/raid:%s: in-place reshape must be started in read-only mode - aborting\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7549) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7550) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7552) } else if (mddev->reshape_backwards
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7553) ? (here_new * chunk_sectors + min_offset_diff <=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7554) here_old * chunk_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7555) : (here_new * chunk_sectors >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7556) here_old * chunk_sectors + (-min_offset_diff))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7557) /* Reading from the same stripe as writing to - bad */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7558) pr_warn("md/raid:%s: reshape_position too early for auto-recovery - aborting.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7559) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7560) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7562) pr_debug("md/raid:%s: reshape will continue\n", mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7563) /* OK, we should be able to continue; */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7564) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7565) BUG_ON(mddev->level != mddev->new_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7566) BUG_ON(mddev->layout != mddev->new_layout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7567) BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7568) BUG_ON(mddev->delta_disks != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7571) if (test_bit(MD_HAS_JOURNAL, &mddev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7572) test_bit(MD_HAS_PPL, &mddev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7573) pr_warn("md/raid:%s: using journal device and PPL not allowed - disabling PPL\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7574) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7575) clear_bit(MD_HAS_PPL, &mddev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7576) clear_bit(MD_HAS_MULTIPLE_PPLS, &mddev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7579) if (mddev->private == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7580) conf = setup_conf(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7581) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7582) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7584) if (IS_ERR(conf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7585) return PTR_ERR(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7587) if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7588) if (!journal_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7589) pr_warn("md/raid:%s: journal disk is missing, force array readonly\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7590) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7591) mddev->ro = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7592) set_disk_ro(mddev->gendisk, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7593) } else if (mddev->recovery_cp == MaxSector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7594) set_bit(MD_JOURNAL_CLEAN, &mddev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7597) conf->min_offset_diff = min_offset_diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7598) mddev->thread = conf->thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7599) conf->thread = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7600) mddev->private = conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7602) for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7603) i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7604) rdev = conf->disks[i].rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7605) if (!rdev && conf->disks[i].replacement) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7606) /* The replacement is all we have yet */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7607) rdev = conf->disks[i].replacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7608) conf->disks[i].replacement = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7609) clear_bit(Replacement, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7610) conf->disks[i].rdev = rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7612) if (!rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7613) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7614) if (conf->disks[i].replacement &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7615) conf->reshape_progress != MaxSector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7616) /* replacements and reshape simply do not mix. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7617) pr_warn("md: cannot handle concurrent replacement and reshape.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7618) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7620) if (test_bit(In_sync, &rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7621) working_disks++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7622) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7624) /* This disc is not fully in-sync. However if it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7625) * just stored parity (beyond the recovery_offset),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7626) * when we don't need to be concerned about the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7627) * array being dirty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7628) * When reshape goes 'backwards', we never have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7629) * partially completed devices, so we only need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7630) * to worry about reshape going forwards.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7631) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7632) /* Hack because v0.91 doesn't store recovery_offset properly. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7633) if (mddev->major_version == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7634) mddev->minor_version > 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7635) rdev->recovery_offset = reshape_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7637) if (rdev->recovery_offset < reshape_offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7638) /* We need to check old and new layout */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7639) if (!only_parity(rdev->raid_disk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7640) conf->algorithm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7641) conf->raid_disks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7642) conf->max_degraded))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7643) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7645) if (!only_parity(rdev->raid_disk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7646) conf->prev_algo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7647) conf->previous_raid_disks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7648) conf->max_degraded))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7649) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7650) dirty_parity_disks++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7653) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7654) * 0 for a fully functional array, 1 or 2 for a degraded array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7655) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7656) mddev->degraded = raid5_calc_degraded(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7658) if (has_failed(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7659) pr_crit("md/raid:%s: not enough operational devices (%d/%d failed)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7660) mdname(mddev), mddev->degraded, conf->raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7661) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7664) /* device size must be a multiple of chunk size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7665) mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7666) mddev->resync_max_sectors = mddev->dev_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7668) if (mddev->degraded > dirty_parity_disks &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7669) mddev->recovery_cp != MaxSector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7670) if (test_bit(MD_HAS_PPL, &mddev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7671) pr_crit("md/raid:%s: starting dirty degraded array with PPL.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7672) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7673) else if (mddev->ok_start_degraded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7674) pr_crit("md/raid:%s: starting dirty degraded array - data corruption possible.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7675) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7676) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7677) pr_crit("md/raid:%s: cannot start dirty degraded array.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7678) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7679) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7683) pr_info("md/raid:%s: raid level %d active with %d out of %d devices, algorithm %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7684) mdname(mddev), conf->level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7685) mddev->raid_disks-mddev->degraded, mddev->raid_disks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7686) mddev->new_layout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7688) print_raid5_conf(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7690) if (conf->reshape_progress != MaxSector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7691) conf->reshape_safe = conf->reshape_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7692) atomic_set(&conf->reshape_stripes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7693) clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7694) clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7695) set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7696) set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7697) mddev->sync_thread = md_register_thread(md_do_sync, mddev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7698) "reshape");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7699) if (!mddev->sync_thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7700) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7703) /* Ok, everything is just fine now */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7704) if (mddev->to_remove == &raid5_attrs_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7705) mddev->to_remove = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7706) else if (mddev->kobj.sd &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7707) sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7708) pr_warn("raid5: failed to create sysfs attributes for %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7709) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7710) md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7712) if (mddev->queue) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7713) int chunk_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7714) /* read-ahead size must cover two whole stripes, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7715) * is 2 * (datadisks) * chunksize where 'n' is the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7716) * number of raid devices
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7717) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7718) int data_disks = conf->previous_raid_disks - conf->max_degraded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7719) int stripe = data_disks *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7720) ((mddev->chunk_sectors << 9) / PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7722) chunk_size = mddev->chunk_sectors << 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7723) blk_queue_io_min(mddev->queue, chunk_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7724) raid5_set_io_opt(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7725) mddev->queue->limits.raid_partial_stripes_expensive = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7726) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7727) * We can only discard a whole stripe. It doesn't make sense to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7728) * discard data disk but write parity disk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7729) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7730) stripe = stripe * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7731) /* Round up to power of 2, as discard handling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7732) * currently assumes that */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7733) while ((stripe-1) & stripe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7734) stripe = (stripe | (stripe-1)) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7735) mddev->queue->limits.discard_alignment = stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7736) mddev->queue->limits.discard_granularity = stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7738) blk_queue_max_write_same_sectors(mddev->queue, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7739) blk_queue_max_write_zeroes_sectors(mddev->queue, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7741) rdev_for_each(rdev, mddev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7742) disk_stack_limits(mddev->gendisk, rdev->bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7743) rdev->data_offset << 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7744) disk_stack_limits(mddev->gendisk, rdev->bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7745) rdev->new_data_offset << 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7748) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7749) * zeroing is required, otherwise data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7750) * could be lost. Consider a scenario: discard a stripe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7751) * (the stripe could be inconsistent if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7752) * discard_zeroes_data is 0); write one disk of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7753) * stripe (the stripe could be inconsistent again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7754) * depending on which disks are used to calculate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7755) * parity); the disk is broken; The stripe data of this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7756) * disk is lost.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7757) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7758) * We only allow DISCARD if the sysadmin has confirmed that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7759) * only safe devices are in use by setting a module parameter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7760) * A better idea might be to turn DISCARD into WRITE_ZEROES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7761) * requests, as that is required to be safe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7762) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7763) if (devices_handle_discard_safely &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7764) mddev->queue->limits.max_discard_sectors >= (stripe >> 9) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7765) mddev->queue->limits.discard_granularity >= stripe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7766) blk_queue_flag_set(QUEUE_FLAG_DISCARD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7767) mddev->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7768) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7769) blk_queue_flag_clear(QUEUE_FLAG_DISCARD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7770) mddev->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7772) blk_queue_max_hw_sectors(mddev->queue, UINT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7775) if (log_init(conf, journal_dev, raid5_has_ppl(conf)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7776) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7778) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7779) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7780) md_unregister_thread(&mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7781) print_raid5_conf(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7782) free_conf(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7783) mddev->private = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7784) pr_warn("md/raid:%s: failed to run raid set.\n", mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7785) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7788) static void raid5_free(struct mddev *mddev, void *priv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7789) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7790) struct r5conf *conf = priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7792) free_conf(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7793) mddev->to_remove = &raid5_attrs_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7794) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7796) static void raid5_status(struct seq_file *seq, struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7797) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7798) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7799) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7801) seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7802) conf->chunk_sectors / 2, mddev->layout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7803) seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7804) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7805) for (i = 0; i < conf->raid_disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7806) struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7807) seq_printf (seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7808) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7809) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7810) seq_printf (seq, "]");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7813) static void print_raid5_conf (struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7814) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7815) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7816) struct disk_info *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7818) pr_debug("RAID conf printout:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7819) if (!conf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7820) pr_debug("(conf==NULL)\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7821) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7823) pr_debug(" --- level:%d rd:%d wd:%d\n", conf->level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7824) conf->raid_disks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7825) conf->raid_disks - conf->mddev->degraded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7827) for (i = 0; i < conf->raid_disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7828) char b[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7829) tmp = conf->disks + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7830) if (tmp->rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7831) pr_debug(" disk %d, o:%d, dev:%s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7832) i, !test_bit(Faulty, &tmp->rdev->flags),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7833) bdevname(tmp->rdev->bdev, b));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7834) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7837) static int raid5_spare_active(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7838) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7839) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7840) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7841) struct disk_info *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7842) int count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7843) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7845) for (i = 0; i < conf->raid_disks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7846) tmp = conf->disks + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7847) if (tmp->replacement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7848) && tmp->replacement->recovery_offset == MaxSector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7849) && !test_bit(Faulty, &tmp->replacement->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7850) && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7851) /* Replacement has just become active. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7852) if (!tmp->rdev
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7853) || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7854) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7855) if (tmp->rdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7856) /* Replaced device not technically faulty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7857) * but we need to be sure it gets removed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7858) * and never re-added.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7859) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7860) set_bit(Faulty, &tmp->rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7861) sysfs_notify_dirent_safe(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7862) tmp->rdev->sysfs_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7864) sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7865) } else if (tmp->rdev
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7866) && tmp->rdev->recovery_offset == MaxSector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7867) && !test_bit(Faulty, &tmp->rdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7868) && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7869) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7870) sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7873) spin_lock_irqsave(&conf->device_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7874) mddev->degraded = raid5_calc_degraded(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7875) spin_unlock_irqrestore(&conf->device_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7876) print_raid5_conf(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7877) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7880) static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7881) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7882) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7883) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7884) int number = rdev->raid_disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7885) struct md_rdev **rdevp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7886) struct disk_info *p = conf->disks + number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7888) print_raid5_conf(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7889) if (test_bit(Journal, &rdev->flags) && conf->log) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7890) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7891) * we can't wait pending write here, as this is called in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7892) * raid5d, wait will deadlock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7893) * neilb: there is no locking about new writes here,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7894) * so this cannot be safe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7896) if (atomic_read(&conf->active_stripes) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7897) atomic_read(&conf->r5c_cached_full_stripes) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7898) atomic_read(&conf->r5c_cached_partial_stripes)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7899) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7901) log_exit(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7902) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7903) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7904) if (rdev == p->rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7905) rdevp = &p->rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7906) else if (rdev == p->replacement)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7907) rdevp = &p->replacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7908) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7909) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7911) if (number >= conf->raid_disks &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7912) conf->reshape_progress == MaxSector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7913) clear_bit(In_sync, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7915) if (test_bit(In_sync, &rdev->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7916) atomic_read(&rdev->nr_pending)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7917) err = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7918) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7920) /* Only remove non-faulty devices if recovery
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7921) * isn't possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7922) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7923) if (!test_bit(Faulty, &rdev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7924) mddev->recovery_disabled != conf->recovery_disabled &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7925) !has_failed(conf) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7926) (!p->replacement || p->replacement == rdev) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7927) number < conf->raid_disks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7928) err = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7929) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7931) *rdevp = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7932) if (!test_bit(RemoveSynchronized, &rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7933) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7934) if (atomic_read(&rdev->nr_pending)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7935) /* lost the race, try later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7936) err = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7937) *rdevp = rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7939) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7940) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7941) err = log_modify(conf, rdev, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7942) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7943) goto abort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7944) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7945) if (p->replacement) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7946) /* We must have just cleared 'rdev' */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7947) p->rdev = p->replacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7948) clear_bit(Replacement, &p->replacement->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7949) smp_mb(); /* Make sure other CPUs may see both as identical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7950) * but will never see neither - if they are careful
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7951) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7952) p->replacement = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7954) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7955) err = log_modify(conf, p->rdev, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7958) clear_bit(WantReplacement, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7959) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7961) print_raid5_conf(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7962) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7963) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7965) static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7966) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7967) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7968) int ret, err = -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7969) int disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7970) struct disk_info *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7971) int first = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7972) int last = conf->raid_disks - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7974) if (test_bit(Journal, &rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7975) if (conf->log)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7976) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7978) rdev->raid_disk = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7979) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7980) * The array is in readonly mode if journal is missing, so no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7981) * write requests running. We should be safe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7982) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7983) ret = log_init(conf, rdev, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7984) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7985) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7987) ret = r5l_start(conf->log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7988) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7989) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7991) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7992) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7993) if (mddev->recovery_disabled == conf->recovery_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7994) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7996) if (rdev->saved_raid_disk < 0 && has_failed(conf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7997) /* no point adding a device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7998) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8000) if (rdev->raid_disk >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8001) first = last = rdev->raid_disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8003) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8004) * find the disk ... but prefer rdev->saved_raid_disk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8005) * if possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8006) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8007) if (rdev->saved_raid_disk >= 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8008) rdev->saved_raid_disk >= first &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8009) conf->disks[rdev->saved_raid_disk].rdev == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8010) first = rdev->saved_raid_disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8012) for (disk = first; disk <= last; disk++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8013) p = conf->disks + disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8014) if (p->rdev == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8015) clear_bit(In_sync, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8016) rdev->raid_disk = disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8017) if (rdev->saved_raid_disk != disk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8018) conf->fullsync = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8019) rcu_assign_pointer(p->rdev, rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8021) err = log_modify(conf, rdev, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8023) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8026) for (disk = first; disk <= last; disk++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8027) p = conf->disks + disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8028) if (test_bit(WantReplacement, &p->rdev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8029) p->replacement == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8030) clear_bit(In_sync, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8031) set_bit(Replacement, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8032) rdev->raid_disk = disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8033) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8034) conf->fullsync = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8035) rcu_assign_pointer(p->replacement, rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8036) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8037) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8038) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8039) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8040) print_raid5_conf(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8041) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8044) static int raid5_resize(struct mddev *mddev, sector_t sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8045) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8046) /* no resync is happening, and there is enough space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8047) * on all devices, so we can resize.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8048) * We need to make sure resync covers any new space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8049) * If the array is shrinking we should possibly wait until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8050) * any io in the removed space completes, but it hardly seems
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8051) * worth it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8052) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8053) sector_t newsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8054) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8056) if (raid5_has_log(conf) || raid5_has_ppl(conf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8057) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8058) sectors &= ~((sector_t)conf->chunk_sectors - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8059) newsize = raid5_size(mddev, sectors, mddev->raid_disks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8060) if (mddev->external_size &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8061) mddev->array_sectors > newsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8062) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8063) if (mddev->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8064) int ret = md_bitmap_resize(mddev->bitmap, sectors, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8065) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8066) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8067) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8068) md_set_array_sectors(mddev, newsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8069) if (sectors > mddev->dev_sectors &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8070) mddev->recovery_cp > mddev->dev_sectors) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8071) mddev->recovery_cp = mddev->dev_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8072) set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8074) mddev->dev_sectors = sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8075) mddev->resync_max_sectors = sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8076) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8077) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8079) static int check_stripe_cache(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8080) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8081) /* Can only proceed if there are plenty of stripe_heads.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8082) * We need a minimum of one full stripe,, and for sensible progress
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8083) * it is best to have about 4 times that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8084) * If we require 4 times, then the default 256 4K stripe_heads will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8085) * allow for chunk sizes up to 256K, which is probably OK.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8086) * If the chunk size is greater, user-space should request more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8087) * stripe_heads first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8088) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8089) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8090) if (((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8091) > conf->min_nr_stripes ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8092) ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8093) > conf->min_nr_stripes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8094) pr_warn("md/raid:%s: reshape: not enough stripes. Needed %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8095) mdname(mddev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8096) ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8097) / RAID5_STRIPE_SIZE(conf))*4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8098) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8099) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8100) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8103) static int check_reshape(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8105) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8107) if (raid5_has_log(conf) || raid5_has_ppl(conf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8108) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8109) if (mddev->delta_disks == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8110) mddev->new_layout == mddev->layout &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8111) mddev->new_chunk_sectors == mddev->chunk_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8112) return 0; /* nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8113) if (has_failed(conf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8114) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8115) if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8116) /* We might be able to shrink, but the devices must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8117) * be made bigger first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8118) * For raid6, 4 is the minimum size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8119) * Otherwise 2 is the minimum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8120) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8121) int min = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8122) if (mddev->level == 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8123) min = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8124) if (mddev->raid_disks + mddev->delta_disks < min)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8125) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8128) if (!check_stripe_cache(mddev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8129) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8131) if (mddev->new_chunk_sectors > mddev->chunk_sectors ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8132) mddev->delta_disks > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8133) if (resize_chunks(conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8134) conf->previous_raid_disks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8135) + max(0, mddev->delta_disks),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8136) max(mddev->new_chunk_sectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8137) mddev->chunk_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8138) ) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8139) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8141) if (conf->previous_raid_disks + mddev->delta_disks <= conf->pool_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8142) return 0; /* never bother to shrink */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8143) return resize_stripes(conf, (conf->previous_raid_disks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8144) + mddev->delta_disks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8147) static int raid5_start_reshape(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8149) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8150) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8151) int spares = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8152) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8154) if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8155) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8157) if (!check_stripe_cache(mddev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8158) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8160) if (has_failed(conf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8161) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8163) rdev_for_each(rdev, mddev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8164) if (!test_bit(In_sync, &rdev->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8165) && !test_bit(Faulty, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8166) spares++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8169) if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8170) /* Not enough devices even to make a degraded array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8171) * of that size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8172) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8173) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8175) /* Refuse to reduce size of the array. Any reductions in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8176) * array size must be through explicit setting of array_size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8177) * attribute.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8178) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8179) if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8180) < mddev->array_sectors) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8181) pr_warn("md/raid:%s: array size must be reduced before number of disks\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8182) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8183) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8186) atomic_set(&conf->reshape_stripes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8187) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8188) write_seqcount_begin(&conf->gen_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8189) conf->previous_raid_disks = conf->raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8190) conf->raid_disks += mddev->delta_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8191) conf->prev_chunk_sectors = conf->chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8192) conf->chunk_sectors = mddev->new_chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8193) conf->prev_algo = conf->algorithm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8194) conf->algorithm = mddev->new_layout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8195) conf->generation++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8196) /* Code that selects data_offset needs to see the generation update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8197) * if reshape_progress has been set - so a memory barrier needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8199) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8200) if (mddev->reshape_backwards)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8201) conf->reshape_progress = raid5_size(mddev, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8202) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8203) conf->reshape_progress = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8204) conf->reshape_safe = conf->reshape_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8205) write_seqcount_end(&conf->gen_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8206) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8208) /* Now make sure any requests that proceeded on the assumption
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8209) * the reshape wasn't running - like Discard or Read - have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8210) * completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8212) mddev_suspend(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8213) mddev_resume(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8215) /* Add some new drives, as many as will fit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8216) * We know there are enough to make the newly sized array work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8217) * Don't add devices if we are reducing the number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8218) * devices in the array. This is because it is not possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8219) * to correctly record the "partially reconstructed" state of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8220) * such devices during the reshape and confusion could result.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8221) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8222) if (mddev->delta_disks >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8223) rdev_for_each(rdev, mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8224) if (rdev->raid_disk < 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8225) !test_bit(Faulty, &rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8226) if (raid5_add_disk(mddev, rdev) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8227) if (rdev->raid_disk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8228) >= conf->previous_raid_disks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8229) set_bit(In_sync, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8230) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8231) rdev->recovery_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8233) /* Failure here is OK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8234) sysfs_link_rdev(mddev, rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8236) } else if (rdev->raid_disk >= conf->previous_raid_disks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8237) && !test_bit(Faulty, &rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8238) /* This is a spare that was manually added */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8239) set_bit(In_sync, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8242) /* When a reshape changes the number of devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8243) * ->degraded is measured against the larger of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8244) * pre and post number of devices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8245) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8246) spin_lock_irqsave(&conf->device_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8247) mddev->degraded = raid5_calc_degraded(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8248) spin_unlock_irqrestore(&conf->device_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8250) mddev->raid_disks = conf->raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8251) mddev->reshape_position = conf->reshape_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8252) set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8254) clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8255) clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8256) clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8257) set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8258) set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8259) mddev->sync_thread = md_register_thread(md_do_sync, mddev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8260) "reshape");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8261) if (!mddev->sync_thread) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8262) mddev->recovery = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8263) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8264) write_seqcount_begin(&conf->gen_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8265) mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8266) mddev->new_chunk_sectors =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8267) conf->chunk_sectors = conf->prev_chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8268) mddev->new_layout = conf->algorithm = conf->prev_algo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8269) rdev_for_each(rdev, mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8270) rdev->new_data_offset = rdev->data_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8271) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8272) conf->generation --;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8273) conf->reshape_progress = MaxSector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8274) mddev->reshape_position = MaxSector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8275) write_seqcount_end(&conf->gen_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8276) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8277) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8279) conf->reshape_checkpoint = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8280) md_wakeup_thread(mddev->sync_thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8281) md_new_event(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8282) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8285) /* This is called from the reshape thread and should make any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8286) * changes needed in 'conf'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8287) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8288) static void end_reshape(struct r5conf *conf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8291) if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8292) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8294) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8295) conf->previous_raid_disks = conf->raid_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8296) md_finish_reshape(conf->mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8297) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8298) conf->reshape_progress = MaxSector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8299) conf->mddev->reshape_position = MaxSector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8300) rdev_for_each(rdev, conf->mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8301) if (rdev->raid_disk >= 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8302) !test_bit(Journal, &rdev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8303) !test_bit(In_sync, &rdev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8304) rdev->recovery_offset = MaxSector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8305) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8306) wake_up(&conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8308) if (conf->mddev->queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8309) raid5_set_io_opt(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8313) /* This is called from the raid5d thread with mddev_lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8314) * It makes config changes to the device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8315) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8316) static void raid5_finish_reshape(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8318) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8320) if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8322) if (mddev->delta_disks <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8323) int d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8324) spin_lock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8325) mddev->degraded = raid5_calc_degraded(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8326) spin_unlock_irq(&conf->device_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8327) for (d = conf->raid_disks ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8328) d < conf->raid_disks - mddev->delta_disks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8329) d++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8330) struct md_rdev *rdev = conf->disks[d].rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8331) if (rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8332) clear_bit(In_sync, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8333) rdev = conf->disks[d].replacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8334) if (rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8335) clear_bit(In_sync, &rdev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8338) mddev->layout = conf->algorithm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8339) mddev->chunk_sectors = conf->chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8340) mddev->reshape_position = MaxSector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8341) mddev->delta_disks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8342) mddev->reshape_backwards = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8346) static void raid5_quiesce(struct mddev *mddev, int quiesce)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8348) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8350) if (quiesce) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8351) /* stop all writes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8352) lock_all_device_hash_locks_irq(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8353) /* '2' tells resync/reshape to pause so that all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8354) * active stripes can drain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8355) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8356) r5c_flush_cache(conf, INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8357) conf->quiesce = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8358) wait_event_cmd(conf->wait_for_quiescent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8359) atomic_read(&conf->active_stripes) == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8360) atomic_read(&conf->active_aligned_reads) == 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8361) unlock_all_device_hash_locks_irq(conf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8362) lock_all_device_hash_locks_irq(conf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8363) conf->quiesce = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8364) unlock_all_device_hash_locks_irq(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8365) /* allow reshape to continue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8366) wake_up(&conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8367) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8368) /* re-enable writes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8369) lock_all_device_hash_locks_irq(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8370) conf->quiesce = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8371) wake_up(&conf->wait_for_quiescent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8372) wake_up(&conf->wait_for_overlap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8373) unlock_all_device_hash_locks_irq(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8375) log_quiesce(conf, quiesce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8378) static void *raid45_takeover_raid0(struct mddev *mddev, int level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8380) struct r0conf *raid0_conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8381) sector_t sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8383) /* for raid0 takeover only one zone is supported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8384) if (raid0_conf->nr_strip_zones > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8385) pr_warn("md/raid:%s: cannot takeover raid0 with more than one zone.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8386) mdname(mddev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8387) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8390) sectors = raid0_conf->strip_zone[0].zone_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8391) sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8392) mddev->dev_sectors = sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8393) mddev->new_level = level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8394) mddev->new_layout = ALGORITHM_PARITY_N;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8395) mddev->new_chunk_sectors = mddev->chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8396) mddev->raid_disks += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8397) mddev->delta_disks = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8398) /* make sure it will be not marked as dirty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8399) mddev->recovery_cp = MaxSector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8401) return setup_conf(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8404) static void *raid5_takeover_raid1(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8406) int chunksect;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8407) void *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8409) if (mddev->raid_disks != 2 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8410) mddev->degraded > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8411) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8413) /* Should check if there are write-behind devices? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8415) chunksect = 64*2; /* 64K by default */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8417) /* The array must be an exact multiple of chunksize */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8418) while (chunksect && (mddev->array_sectors & (chunksect-1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8419) chunksect >>= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8421) if ((chunksect<<9) < RAID5_STRIPE_SIZE((struct r5conf *)mddev->private))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8422) /* array size does not allow a suitable chunk size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8423) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8425) mddev->new_level = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8426) mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8427) mddev->new_chunk_sectors = chunksect;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8429) ret = setup_conf(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8430) if (!IS_ERR(ret))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8431) mddev_clear_unsupported_flags(mddev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8432) UNSUPPORTED_MDDEV_FLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8433) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8436) static void *raid5_takeover_raid6(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8438) int new_layout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8440) switch (mddev->layout) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8441) case ALGORITHM_LEFT_ASYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8442) new_layout = ALGORITHM_LEFT_ASYMMETRIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8443) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8444) case ALGORITHM_RIGHT_ASYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8445) new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8446) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8447) case ALGORITHM_LEFT_SYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8448) new_layout = ALGORITHM_LEFT_SYMMETRIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8449) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8450) case ALGORITHM_RIGHT_SYMMETRIC_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8451) new_layout = ALGORITHM_RIGHT_SYMMETRIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8452) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8453) case ALGORITHM_PARITY_0_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8454) new_layout = ALGORITHM_PARITY_0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8455) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8456) case ALGORITHM_PARITY_N:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8457) new_layout = ALGORITHM_PARITY_N;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8458) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8459) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8460) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8462) mddev->new_level = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8463) mddev->new_layout = new_layout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8464) mddev->delta_disks = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8465) mddev->raid_disks -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8466) return setup_conf(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8469) static int raid5_check_reshape(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8470) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8471) /* For a 2-drive array, the layout and chunk size can be changed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8472) * immediately as not restriping is needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8473) * For larger arrays we record the new value - after validation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8474) * to be used by a reshape pass.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8475) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8476) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8477) int new_chunk = mddev->new_chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8479) if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8480) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8481) if (new_chunk > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8482) if (!is_power_of_2(new_chunk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8483) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8484) if (new_chunk < (PAGE_SIZE>>9))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8485) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8486) if (mddev->array_sectors & (new_chunk-1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8487) /* not factor of array size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8488) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8491) /* They look valid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8493) if (mddev->raid_disks == 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8494) /* can make the change immediately */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8495) if (mddev->new_layout >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8496) conf->algorithm = mddev->new_layout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8497) mddev->layout = mddev->new_layout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8499) if (new_chunk > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8500) conf->chunk_sectors = new_chunk ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8501) mddev->chunk_sectors = new_chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8503) set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8504) md_wakeup_thread(mddev->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8506) return check_reshape(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8509) static int raid6_check_reshape(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8511) int new_chunk = mddev->new_chunk_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8513) if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8514) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8515) if (new_chunk > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8516) if (!is_power_of_2(new_chunk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8517) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8518) if (new_chunk < (PAGE_SIZE >> 9))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8519) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8520) if (mddev->array_sectors & (new_chunk-1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8521) /* not factor of array size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8522) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8525) /* They look valid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8526) return check_reshape(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8529) static void *raid5_takeover(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8530) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8531) /* raid5 can take over:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8532) * raid0 - if there is only one strip zone - make it a raid4 layout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8533) * raid1 - if there are two drives. We need to know the chunk size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8534) * raid4 - trivial - just use a raid4 layout.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8535) * raid6 - Providing it is a *_6 layout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8536) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8537) if (mddev->level == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8538) return raid45_takeover_raid0(mddev, 5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8539) if (mddev->level == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8540) return raid5_takeover_raid1(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8541) if (mddev->level == 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8542) mddev->new_layout = ALGORITHM_PARITY_N;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8543) mddev->new_level = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8544) return setup_conf(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8546) if (mddev->level == 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8547) return raid5_takeover_raid6(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8549) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8552) static void *raid4_takeover(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8553) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8554) /* raid4 can take over:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8555) * raid0 - if there is only one strip zone
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8556) * raid5 - if layout is right
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8557) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8558) if (mddev->level == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8559) return raid45_takeover_raid0(mddev, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8560) if (mddev->level == 5 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8561) mddev->layout == ALGORITHM_PARITY_N) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8562) mddev->new_layout = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8563) mddev->new_level = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8564) return setup_conf(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8566) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8569) static struct md_personality raid5_personality;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8571) static void *raid6_takeover(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8572) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8573) /* Currently can only take over a raid5. We map the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8574) * personality to an equivalent raid6 personality
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8575) * with the Q block at the end.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8576) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8577) int new_layout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8579) if (mddev->pers != &raid5_personality)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8580) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8581) if (mddev->degraded > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8582) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8583) if (mddev->raid_disks > 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8584) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8585) if (mddev->raid_disks < 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8586) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8588) switch (mddev->layout) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8589) case ALGORITHM_LEFT_ASYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8590) new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8591) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8592) case ALGORITHM_RIGHT_ASYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8593) new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8594) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8595) case ALGORITHM_LEFT_SYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8596) new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8597) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8598) case ALGORITHM_RIGHT_SYMMETRIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8599) new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8600) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8601) case ALGORITHM_PARITY_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8602) new_layout = ALGORITHM_PARITY_0_6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8603) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8604) case ALGORITHM_PARITY_N:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8605) new_layout = ALGORITHM_PARITY_N;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8606) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8607) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8608) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8610) mddev->new_level = 6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8611) mddev->new_layout = new_layout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8612) mddev->delta_disks = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8613) mddev->raid_disks += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8614) return setup_conf(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8617) static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8618) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8619) struct r5conf *conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8620) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8622) err = mddev_lock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8623) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8624) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8625) conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8626) if (!conf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8627) mddev_unlock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8628) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8631) if (strncmp(buf, "ppl", 3) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8632) /* ppl only works with RAID 5 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8633) if (!raid5_has_ppl(conf) && conf->level == 5) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8634) err = log_init(conf, NULL, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8635) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8636) err = resize_stripes(conf, conf->pool_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8637) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8638) log_exit(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8640) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8641) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8642) } else if (strncmp(buf, "resync", 6) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8643) if (raid5_has_ppl(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8644) mddev_suspend(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8645) log_exit(conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8646) mddev_resume(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8647) err = resize_stripes(conf, conf->pool_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8648) } else if (test_bit(MD_HAS_JOURNAL, &conf->mddev->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8649) r5l_log_disk_error(conf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8650) bool journal_dev_exists = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8651) struct md_rdev *rdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8653) rdev_for_each(rdev, mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8654) if (test_bit(Journal, &rdev->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8655) journal_dev_exists = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8656) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8659) if (!journal_dev_exists) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8660) mddev_suspend(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8661) clear_bit(MD_HAS_JOURNAL, &mddev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8662) mddev_resume(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8663) } else /* need remove journal device first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8664) err = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8665) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8666) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8667) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8668) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8671) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8672) md_update_sb(mddev, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8674) mddev_unlock(mddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8676) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8679) static int raid5_start(struct mddev *mddev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8680) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8681) struct r5conf *conf = mddev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8683) return r5l_start(conf->log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8686) static struct md_personality raid6_personality =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8687) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8688) .name = "raid6",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8689) .level = 6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8690) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8691) .make_request = raid5_make_request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8692) .run = raid5_run,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8693) .start = raid5_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8694) .free = raid5_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8695) .status = raid5_status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8696) .error_handler = raid5_error,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8697) .hot_add_disk = raid5_add_disk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8698) .hot_remove_disk= raid5_remove_disk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8699) .spare_active = raid5_spare_active,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8700) .sync_request = raid5_sync_request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8701) .resize = raid5_resize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8702) .size = raid5_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8703) .check_reshape = raid6_check_reshape,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8704) .start_reshape = raid5_start_reshape,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8705) .finish_reshape = raid5_finish_reshape,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8706) .quiesce = raid5_quiesce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8707) .takeover = raid6_takeover,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8708) .change_consistency_policy = raid5_change_consistency_policy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8709) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8710) static struct md_personality raid5_personality =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8711) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8712) .name = "raid5",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8713) .level = 5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8714) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8715) .make_request = raid5_make_request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8716) .run = raid5_run,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8717) .start = raid5_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8718) .free = raid5_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8719) .status = raid5_status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8720) .error_handler = raid5_error,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8721) .hot_add_disk = raid5_add_disk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8722) .hot_remove_disk= raid5_remove_disk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8723) .spare_active = raid5_spare_active,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8724) .sync_request = raid5_sync_request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8725) .resize = raid5_resize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8726) .size = raid5_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8727) .check_reshape = raid5_check_reshape,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8728) .start_reshape = raid5_start_reshape,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8729) .finish_reshape = raid5_finish_reshape,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8730) .quiesce = raid5_quiesce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8731) .takeover = raid5_takeover,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8732) .change_consistency_policy = raid5_change_consistency_policy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8733) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8735) static struct md_personality raid4_personality =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8736) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8737) .name = "raid4",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8738) .level = 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8739) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8740) .make_request = raid5_make_request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8741) .run = raid5_run,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8742) .start = raid5_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8743) .free = raid5_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8744) .status = raid5_status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8745) .error_handler = raid5_error,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8746) .hot_add_disk = raid5_add_disk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8747) .hot_remove_disk= raid5_remove_disk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8748) .spare_active = raid5_spare_active,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8749) .sync_request = raid5_sync_request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8750) .resize = raid5_resize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8751) .size = raid5_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8752) .check_reshape = raid5_check_reshape,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8753) .start_reshape = raid5_start_reshape,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8754) .finish_reshape = raid5_finish_reshape,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8755) .quiesce = raid5_quiesce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8756) .takeover = raid4_takeover,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8757) .change_consistency_policy = raid5_change_consistency_policy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8758) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8760) static int __init raid5_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8761) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8762) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8764) raid5_wq = alloc_workqueue("raid5wq",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8765) WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8766) if (!raid5_wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8767) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8769) ret = cpuhp_setup_state_multi(CPUHP_MD_RAID5_PREPARE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8770) "md/raid5:prepare",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8771) raid456_cpu_up_prepare,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8772) raid456_cpu_dead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8773) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8774) destroy_workqueue(raid5_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8775) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8777) register_md_personality(&raid6_personality);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8778) register_md_personality(&raid5_personality);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8779) register_md_personality(&raid4_personality);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8780) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8781) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8783) static void raid5_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8784) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8785) unregister_md_personality(&raid6_personality);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8786) unregister_md_personality(&raid5_personality);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8787) unregister_md_personality(&raid4_personality);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8788) cpuhp_remove_multi_state(CPUHP_MD_RAID5_PREPARE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8789) destroy_workqueue(raid5_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8792) module_init(raid5_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8793) module_exit(raid5_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8794) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8795) MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8796) MODULE_ALIAS("md-personality-4"); /* RAID5 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8797) MODULE_ALIAS("md-raid5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8798) MODULE_ALIAS("md-raid4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8799) MODULE_ALIAS("md-level-5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8800) MODULE_ALIAS("md-level-4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8801) MODULE_ALIAS("md-personality-8"); /* RAID6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8802) MODULE_ALIAS("md-raid6");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8803) MODULE_ALIAS("md-level-6");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8805) /* This used to be two separate modules, they were: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8806) MODULE_ALIAS("raid5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8807) MODULE_ALIAS("raid6");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags