^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright(c) 2017 Intel Corporation. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/mount.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/pseudo_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/magic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/genhd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/pfn_t.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/cdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/uio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/dax.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "dax-private.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) static dev_t dax_devt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) DEFINE_STATIC_SRCU(dax_srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) static struct vfsmount *dax_mnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) static DEFINE_IDA(dax_minor_ida);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) static struct kmem_cache *dax_cache __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) static struct super_block *dax_superblock __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define DAX_HASH_SIZE (PAGE_SIZE / sizeof(struct hlist_head))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static struct hlist_head dax_host_list[DAX_HASH_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) static DEFINE_SPINLOCK(dax_host_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) int dax_read_lock(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) return srcu_read_lock(&dax_srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) EXPORT_SYMBOL_GPL(dax_read_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) void dax_read_unlock(int id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) srcu_read_unlock(&dax_srcu, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) EXPORT_SYMBOL_GPL(dax_read_unlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #ifdef CONFIG_BLOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) int bdev_dax_pgoff(struct block_device *bdev, sector_t sector, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) pgoff_t *pgoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) sector_t start_sect = bdev ? get_start_sect(bdev) : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) phys_addr_t phys_off = (start_sect + sector) * 512;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) if (pgoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) *pgoff = PHYS_PFN(phys_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) if (phys_off % PAGE_SIZE || size % PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) EXPORT_SYMBOL(bdev_dax_pgoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #if IS_ENABLED(CONFIG_FS_DAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) if (!blk_queue_dax(bdev->bd_disk->queue))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) return dax_get_by_host(bdev->bd_disk->disk_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) bool __generic_fsdax_supported(struct dax_device *dax_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) struct block_device *bdev, int blocksize, sector_t start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) sector_t sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) bool dax_enabled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) pgoff_t pgoff, pgoff_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) char buf[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) void *kaddr, *end_kaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) pfn_t pfn, end_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) sector_t last_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) long len, len2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) int err, id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) if (blocksize != PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) pr_info("%s: error: unsupported blocksize for dax\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) bdevname(bdev, buf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (!dax_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) pr_debug("%s: error: dax unsupported by block device\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) bdevname(bdev, buf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) err = bdev_dax_pgoff(bdev, start, PAGE_SIZE, &pgoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) pr_info("%s: error: unaligned partition for dax\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) bdevname(bdev, buf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) last_page = PFN_DOWN((start + sectors - 1) * 512) * PAGE_SIZE / 512;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) err = bdev_dax_pgoff(bdev, last_page, PAGE_SIZE, &pgoff_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) pr_info("%s: error: unaligned partition for dax\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) bdevname(bdev, buf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) id = dax_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) len = dax_direct_access(dax_dev, pgoff, 1, &kaddr, &pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) len2 = dax_direct_access(dax_dev, pgoff_end, 1, &end_kaddr, &end_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (len < 1 || len2 < 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) pr_info("%s: error: dax access failed (%ld)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) bdevname(bdev, buf), len < 1 ? len : len2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) dax_read_unlock(id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (IS_ENABLED(CONFIG_FS_DAX_LIMITED) && pfn_t_special(pfn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * An arch that has enabled the pmem api should also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * have its drivers support pfn_t_devmap()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * This is a developer warning and should not trigger in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * production. dax_flush() will crash since it depends
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * on being able to do (page_address(pfn_to_page())).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) WARN_ON(IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) dax_enabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) } else if (pfn_t_devmap(pfn) && pfn_t_devmap(end_pfn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct dev_pagemap *pgmap, *end_pgmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) pgmap = get_dev_pagemap(pfn_t_to_pfn(pfn), NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) end_pgmap = get_dev_pagemap(pfn_t_to_pfn(end_pfn), NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) if (pgmap && pgmap == end_pgmap && pgmap->type == MEMORY_DEVICE_FS_DAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) && pfn_t_to_page(pfn)->pgmap == pgmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) && pfn_t_to_page(end_pfn)->pgmap == pgmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) && pfn_t_to_pfn(pfn) == PHYS_PFN(__pa(kaddr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) && pfn_t_to_pfn(end_pfn) == PHYS_PFN(__pa(end_kaddr)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) dax_enabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) put_dev_pagemap(pgmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) put_dev_pagemap(end_pgmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) dax_read_unlock(id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) if (!dax_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) pr_info("%s: error: dax support not enabled\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) bdevname(bdev, buf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) EXPORT_SYMBOL_GPL(__generic_fsdax_supported);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * __bdev_dax_supported() - Check if the device supports dax for filesystem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * @bdev: block device to check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * @blocksize: The block size of the device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * This is a library function for filesystems to check if the block device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * can be mounted with dax option.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * Return: true if supported, false if unsupported
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) bool __bdev_dax_supported(struct block_device *bdev, int blocksize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) struct dax_device *dax_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) struct request_queue *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) char buf[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) bool ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) int id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) q = bdev_get_queue(bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (!q || !blk_queue_dax(q)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) pr_debug("%s: error: request queue doesn't support dax\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) bdevname(bdev, buf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (!dax_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) pr_debug("%s: error: device does not support dax\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) bdevname(bdev, buf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) id = dax_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) ret = dax_supported(dax_dev, bdev, blocksize, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) i_size_read(bdev->bd_inode) / 512);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) dax_read_unlock(id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) put_dax(dax_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) EXPORT_SYMBOL_GPL(__bdev_dax_supported);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) enum dax_device_flags {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /* !alive + rcu grace period == no new operations / mappings */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) DAXDEV_ALIVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) /* gate whether dax_flush() calls the low level flush routine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) DAXDEV_WRITE_CACHE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) /* flag to check if device supports synchronous flush */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) DAXDEV_SYNC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * struct dax_device - anchor object for dax services
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * @inode: core vfs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * @cdev: optional character interface for "device dax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * @host: optional name for lookups where the device path is not available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * @private: dax driver private data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * @flags: state and boolean properties
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) struct dax_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) struct hlist_node list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) struct inode inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) struct cdev cdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) const char *host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) void *private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) const struct dax_operations *ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) static ssize_t write_cache_show(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) struct dax_device *dax_dev = dax_get_by_host(dev_name(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) ssize_t rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) WARN_ON_ONCE(!dax_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) if (!dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) rc = sprintf(buf, "%d\n", !!dax_write_cache_enabled(dax_dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) put_dax(dax_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) static ssize_t write_cache_store(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) struct device_attribute *attr, const char *buf, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) bool write_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) int rc = strtobool(buf, &write_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct dax_device *dax_dev = dax_get_by_host(dev_name(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) WARN_ON_ONCE(!dax_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) if (!dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) len = rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) dax_write_cache(dax_dev, write_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) put_dax(dax_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static DEVICE_ATTR_RW(write_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) static umode_t dax_visible(struct kobject *kobj, struct attribute *a, int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) struct device *dev = container_of(kobj, typeof(*dev), kobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct dax_device *dax_dev = dax_get_by_host(dev_name(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) WARN_ON_ONCE(!dax_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (!dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) #ifndef CONFIG_ARCH_HAS_PMEM_API
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (a == &dev_attr_write_cache.attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) return a->mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static struct attribute *dax_attributes[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) &dev_attr_write_cache.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) struct attribute_group dax_attribute_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) .name = "dax",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) .attrs = dax_attributes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) .is_visible = dax_visible,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) EXPORT_SYMBOL_GPL(dax_attribute_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * dax_direct_access() - translate a device pgoff to an absolute pfn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * @dax_dev: a dax_device instance representing the logical memory range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * @pgoff: offset in pages from the start of the device to translate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * @nr_pages: number of consecutive pages caller can handle relative to @pfn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * @kaddr: output parameter that returns a virtual address mapping of pfn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * @pfn: output parameter that returns an absolute pfn translation of @pgoff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * Return: negative errno if an error occurs, otherwise the number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * pages accessible at the device relative @pgoff.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) void **kaddr, pfn_t *pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) long avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (!dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (!dax_alive(dax_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (nr_pages < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) return nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) kaddr, pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (!avail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return -ERANGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) return min(avail, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) EXPORT_SYMBOL_GPL(dax_direct_access);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) bool dax_supported(struct dax_device *dax_dev, struct block_device *bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) int blocksize, sector_t start, sector_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (!dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (!dax_alive(dax_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return dax_dev->ops->dax_supported(dax_dev, bdev, blocksize, start, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) EXPORT_SYMBOL_GPL(dax_supported);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) size_t bytes, struct iov_iter *i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (!dax_alive(dax_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) return dax_dev->ops->copy_from_iter(dax_dev, pgoff, addr, bytes, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) EXPORT_SYMBOL_GPL(dax_copy_from_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) size_t bytes, struct iov_iter *i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (!dax_alive(dax_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return dax_dev->ops->copy_to_iter(dax_dev, pgoff, addr, bytes, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) EXPORT_SYMBOL_GPL(dax_copy_to_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) size_t nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (!dax_alive(dax_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) * There are no callers that want to zero more than one page as of now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * Once users are there, this check can be removed after the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * device mapper code has been updated to split ranges across targets.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) if (nr_pages != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) EXPORT_SYMBOL_GPL(dax_zero_page_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) #ifdef CONFIG_ARCH_HAS_PMEM_API
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) void arch_wb_cache_pmem(void *addr, size_t size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (unlikely(!dax_write_cache_enabled(dax_dev)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) arch_wb_cache_pmem(addr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) EXPORT_SYMBOL_GPL(dax_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) void dax_write_cache(struct dax_device *dax_dev, bool wc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) if (wc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) EXPORT_SYMBOL_GPL(dax_write_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) bool dax_write_cache_enabled(struct dax_device *dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) EXPORT_SYMBOL_GPL(dax_write_cache_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) bool __dax_synchronous(struct dax_device *dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) return test_bit(DAXDEV_SYNC, &dax_dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) EXPORT_SYMBOL_GPL(__dax_synchronous);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) void __set_dax_synchronous(struct dax_device *dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) set_bit(DAXDEV_SYNC, &dax_dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) EXPORT_SYMBOL_GPL(__set_dax_synchronous);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) bool dax_alive(struct dax_device *dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) lockdep_assert_held(&dax_srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) return test_bit(DAXDEV_ALIVE, &dax_dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) EXPORT_SYMBOL_GPL(dax_alive);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) static int dax_host_hash(const char *host)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) return hashlen_hash(hashlen_string("DAX", host)) % DAX_HASH_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * that any fault handlers or operations that might have seen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * dax_alive(), have completed. Any operations that start after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) * synchronize_srcu() has run will abort upon seeing !dax_alive().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) void kill_dax(struct dax_device *dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (!dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) clear_bit(DAXDEV_ALIVE, &dax_dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) synchronize_srcu(&dax_srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) spin_lock(&dax_host_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) hlist_del_init(&dax_dev->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) spin_unlock(&dax_host_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) EXPORT_SYMBOL_GPL(kill_dax);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) void run_dax(struct dax_device *dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) set_bit(DAXDEV_ALIVE, &dax_dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) EXPORT_SYMBOL_GPL(run_dax);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) static struct inode *dax_alloc_inode(struct super_block *sb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) struct dax_device *dax_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) struct inode *inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) dax_dev = kmem_cache_alloc(dax_cache, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if (!dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) inode = &dax_dev->inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) inode->i_rdev = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) return inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) static struct dax_device *to_dax_dev(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) return container_of(inode, struct dax_device, inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) static void dax_free_inode(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) struct dax_device *dax_dev = to_dax_dev(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) kfree(dax_dev->host);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) dax_dev->host = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) if (inode->i_rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) ida_simple_remove(&dax_minor_ida, MINOR(inode->i_rdev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) kmem_cache_free(dax_cache, dax_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) static void dax_destroy_inode(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) struct dax_device *dax_dev = to_dax_dev(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) "kill_dax() must be called before final iput()\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) static const struct super_operations dax_sops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) .statfs = simple_statfs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) .alloc_inode = dax_alloc_inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) .destroy_inode = dax_destroy_inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) .free_inode = dax_free_inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) .drop_inode = generic_delete_inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) static int dax_init_fs_context(struct fs_context *fc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) struct pseudo_fs_context *ctx = init_pseudo(fc, DAXFS_MAGIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (!ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) ctx->ops = &dax_sops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) static struct file_system_type dax_fs_type = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) .name = "dax",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) .init_fs_context = dax_init_fs_context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) .kill_sb = kill_anon_super,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) static int dax_test(struct inode *inode, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) dev_t devt = *(dev_t *) data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return inode->i_rdev == devt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) static int dax_set(struct inode *inode, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) dev_t devt = *(dev_t *) data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) inode->i_rdev = devt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) static struct dax_device *dax_dev_get(dev_t devt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) struct dax_device *dax_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) struct inode *inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) dax_test, dax_set, &devt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (!inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) dax_dev = to_dax_dev(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) if (inode->i_state & I_NEW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) set_bit(DAXDEV_ALIVE, &dax_dev->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) inode->i_cdev = &dax_dev->cdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) inode->i_mode = S_IFCHR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) inode->i_flags = S_DAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) mapping_set_gfp_mask(&inode->i_data, GFP_USER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) unlock_new_inode(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) return dax_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) static void dax_add_host(struct dax_device *dax_dev, const char *host)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) int hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * Unconditionally init dax_dev since it's coming from a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * non-zeroed slab cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) INIT_HLIST_NODE(&dax_dev->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) dax_dev->host = host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (!host)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) hash = dax_host_hash(host);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) spin_lock(&dax_host_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) hlist_add_head(&dax_dev->list, &dax_host_list[hash]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) spin_unlock(&dax_host_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) struct dax_device *alloc_dax(void *private, const char *__host,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) const struct dax_operations *ops, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) struct dax_device *dax_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) const char *host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) dev_t devt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) int minor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (ops && !ops->zero_page_range) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) pr_debug("%s: error: device does not provide dax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) " operation zero_page_range()\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) __host ? __host : "Unknown");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) host = kstrdup(__host, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) if (__host && !host)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) minor = ida_simple_get(&dax_minor_ida, 0, MINORMASK+1, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) if (minor < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) goto err_minor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) devt = MKDEV(MAJOR(dax_devt), minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) dax_dev = dax_dev_get(devt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (!dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) goto err_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) dax_add_host(dax_dev, host);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) dax_dev->ops = ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) dax_dev->private = private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) if (flags & DAXDEV_F_SYNC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) set_dax_synchronous(dax_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) return dax_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) err_dev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) ida_simple_remove(&dax_minor_ida, minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) err_minor:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) kfree(host);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) EXPORT_SYMBOL_GPL(alloc_dax);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) void put_dax(struct dax_device *dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) if (!dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) iput(&dax_dev->inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) EXPORT_SYMBOL_GPL(put_dax);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) * dax_get_by_host() - temporary lookup mechanism for filesystem-dax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * @host: alternate name for the device registered by a dax driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) struct dax_device *dax_get_by_host(const char *host)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) struct dax_device *dax_dev, *found = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) int hash, id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) if (!host)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) hash = dax_host_hash(host);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) id = dax_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) spin_lock(&dax_host_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) hlist_for_each_entry(dax_dev, &dax_host_list[hash], list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) if (!dax_alive(dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) || strcmp(host, dax_dev->host) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) if (igrab(&dax_dev->inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) found = dax_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) spin_unlock(&dax_host_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) dax_read_unlock(id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) return found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) EXPORT_SYMBOL_GPL(dax_get_by_host);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) * inode_dax: convert a public inode into its dax_dev
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * @inode: An inode with i_cdev pointing to a dax_dev
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) * Note this is not equivalent to to_dax_dev() which is for private
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * internal use where we know the inode filesystem type == dax_fs_type.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) struct dax_device *inode_dax(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) struct cdev *cdev = inode->i_cdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) return container_of(cdev, struct dax_device, cdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) EXPORT_SYMBOL_GPL(inode_dax);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) struct inode *dax_inode(struct dax_device *dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) return &dax_dev->inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) EXPORT_SYMBOL_GPL(dax_inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) void *dax_get_private(struct dax_device *dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) return dax_dev->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) EXPORT_SYMBOL_GPL(dax_get_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) static void init_once(void *_dax_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) struct dax_device *dax_dev = _dax_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) struct inode *inode = &dax_dev->inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) memset(dax_dev, 0, sizeof(*dax_dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) inode_init_once(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) static int dax_fs_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) SLAB_MEM_SPREAD|SLAB_ACCOUNT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) init_once);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) if (!dax_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) dax_mnt = kern_mount(&dax_fs_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) if (IS_ERR(dax_mnt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) rc = PTR_ERR(dax_mnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) goto err_mount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) dax_superblock = dax_mnt->mnt_sb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) err_mount:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) kmem_cache_destroy(dax_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) static void dax_fs_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) kern_unmount(dax_mnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) rcu_barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) kmem_cache_destroy(dax_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) static int __init dax_core_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) rc = dax_fs_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) rc = alloc_chrdev_region(&dax_devt, 0, MINORMASK+1, "dax");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) goto err_chrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) rc = dax_bus_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) goto err_bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) err_bus:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) unregister_chrdev_region(dax_devt, MINORMASK+1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) err_chrdev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) dax_fs_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) static void __exit dax_core_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) dax_bus_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) unregister_chrdev_region(dax_devt, MINORMASK+1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) ida_destroy(&dax_minor_ida);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) dax_fs_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) MODULE_AUTHOR("Intel Corporation");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) MODULE_LICENSE("GPL v2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) subsys_initcall(dax_core_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) module_exit(dax_core_exit);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags