^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Public API and common code for kernel->userspace relay file support.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * See Documentation/filesystems/relay.rst for an overview.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2002-2005 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 1999-2005 - Karim Yaghmour (karim@opersys.com)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Moved to kernel/relay.c by Paul Mundt, 2006.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * November 2006 - CPU hotplug support by Mathieu Desnoyers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * (mathieu.desnoyers@polymtl.ca)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * This file is released under the GPL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/stddef.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/relay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/splice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /* list of open channels, for cpu hotplug */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static DEFINE_MUTEX(relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static LIST_HEAD(relay_channels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * close() vm_op implementation for relay file mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) static void relay_file_mmap_close(struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) struct rchan_buf *buf = vma->vm_private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) buf->chan->cb->buf_unmapped(buf, vma->vm_file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * fault() vm_op implementation for relay file mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static vm_fault_t relay_buf_fault(struct vm_fault *vmf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct rchan_buf *buf = vmf->vma->vm_private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) pgoff_t pgoff = vmf->pgoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) return VM_FAULT_OOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) page = vmalloc_to_page(buf->start + (pgoff << PAGE_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) return VM_FAULT_SIGBUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) vmf->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * vm_ops for relay file mappings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) static const struct vm_operations_struct relay_file_mmap_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) .fault = relay_buf_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) .close = relay_file_mmap_close,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * allocate an array of pointers of struct page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static struct page **relay_alloc_page_array(unsigned int n_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) const size_t pa_size = n_pages * sizeof(struct page *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) if (pa_size > PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) return vzalloc(pa_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return kzalloc(pa_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * free an array of pointers of struct page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static void relay_free_page_array(struct page **array)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) kvfree(array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * relay_mmap_buf: - mmap channel buffer to process address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * @buf: relay channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * @vma: vm_area_struct describing memory to be mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * Returns 0 if ok, negative on error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * Caller should already have grabbed mmap_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) unsigned long length = vma->vm_end - vma->vm_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) struct file *filp = vma->vm_file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (length != (unsigned long)buf->chan->alloc_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) vma->vm_ops = &relay_file_mmap_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) vma->vm_flags |= VM_DONTEXPAND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) vma->vm_private_data = buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) buf->chan->cb->buf_mapped(buf, filp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * relay_alloc_buf - allocate a channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * @buf: the buffer struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * @size: total size of the buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * Returns a pointer to the resulting buffer, %NULL if unsuccessful. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * passed in size will get page aligned, if it isn't already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) void *mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) unsigned int i, j, n_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) *size = PAGE_ALIGN(*size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) n_pages = *size >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) buf->page_array = relay_alloc_page_array(n_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (!buf->page_array)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) for (i = 0; i < n_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) buf->page_array[i] = alloc_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) if (unlikely(!buf->page_array[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) goto depopulate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) set_page_private(buf->page_array[i], (unsigned long)buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (!mem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) goto depopulate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) memset(mem, 0, *size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) buf->page_count = n_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) return mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) depopulate:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) for (j = 0; j < i; j++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) __free_page(buf->page_array[j]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) relay_free_page_array(buf->page_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * relay_create_buf - allocate and initialize a channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * @chan: the relay channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * Returns channel buffer if successful, %NULL otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) static struct rchan_buf *relay_create_buf(struct rchan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) struct rchan_buf *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) if (chan->n_subbufs > KMALLOC_MAX_SIZE / sizeof(size_t *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) buf->padding = kmalloc_array(chan->n_subbufs, sizeof(size_t *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (!buf->padding)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) goto free_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) buf->start = relay_alloc_buf(buf, &chan->alloc_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (!buf->start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) goto free_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) buf->chan = chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) kref_get(&buf->chan->kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) return buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) free_buf:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) kfree(buf->padding);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) kfree(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * relay_destroy_channel - free the channel struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * @kref: target kernel reference that contains the relay channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * Should only be called from kref_put().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) static void relay_destroy_channel(struct kref *kref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) struct rchan *chan = container_of(kref, struct rchan, kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) free_percpu(chan->buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) kfree(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * relay_destroy_buf - destroy an rchan_buf struct and associated buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * @buf: the buffer struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) static void relay_destroy_buf(struct rchan_buf *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct rchan *chan = buf->chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) if (likely(buf->start)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) vunmap(buf->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) for (i = 0; i < buf->page_count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) __free_page(buf->page_array[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) relay_free_page_array(buf->page_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) *per_cpu_ptr(chan->buf, buf->cpu) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) kfree(buf->padding);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) kfree(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) kref_put(&chan->kref, relay_destroy_channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * relay_remove_buf - remove a channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * @kref: target kernel reference that contains the relay buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) * Removes the file from the filesystem, which also frees the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * rchan_buf_struct and the channel buffer. Should only be called from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * kref_put().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) static void relay_remove_buf(struct kref *kref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) relay_destroy_buf(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * relay_buf_empty - boolean, is the channel buffer empty?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * @buf: channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * Returns 1 if the buffer is empty, 0 otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) static int relay_buf_empty(struct rchan_buf *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * relay_buf_full - boolean, is the channel buffer full?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * @buf: channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * Returns 1 if the buffer is full, 0 otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) int relay_buf_full(struct rchan_buf *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) size_t ready = buf->subbufs_produced - buf->subbufs_consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) return (ready >= buf->chan->n_subbufs) ? 1 : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) EXPORT_SYMBOL_GPL(relay_buf_full);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * High-level relay kernel API and associated functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * rchan_callback implementations defining default channel behavior. Used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * in place of corresponding NULL values in client callback struct.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * subbuf_start() default callback. Does nothing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) static int subbuf_start_default_callback (struct rchan_buf *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) void *subbuf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) void *prev_subbuf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) size_t prev_padding)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) if (relay_buf_full(buf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * buf_mapped() default callback. Does nothing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static void buf_mapped_default_callback(struct rchan_buf *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) struct file *filp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * buf_unmapped() default callback. Does nothing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) static void buf_unmapped_default_callback(struct rchan_buf *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) struct file *filp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * create_buf_file_create() default callback. Does nothing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) static struct dentry *create_buf_file_default_callback(const char *filename,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) struct dentry *parent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) umode_t mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) struct rchan_buf *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) int *is_global)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * remove_buf_file() default callback. Does nothing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static int remove_buf_file_default_callback(struct dentry *dentry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /* relay channel default callbacks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) static struct rchan_callbacks default_channel_callbacks = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) .subbuf_start = subbuf_start_default_callback,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) .buf_mapped = buf_mapped_default_callback,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) .buf_unmapped = buf_unmapped_default_callback,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) .create_buf_file = create_buf_file_default_callback,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) .remove_buf_file = remove_buf_file_default_callback,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * wakeup_readers - wake up readers waiting on a channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * @work: contains the channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) * This is the function used to defer reader waking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) static void wakeup_readers(struct irq_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) struct rchan_buf *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) buf = container_of(work, struct rchan_buf, wakeup_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) wake_up_interruptible(&buf->read_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * __relay_reset - reset a channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * @buf: the channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * @init: 1 if this is a first-time initialization
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * See relay_reset() for description of effect.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) static void __relay_reset(struct rchan_buf *buf, unsigned int init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) size_t i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (init) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) init_waitqueue_head(&buf->read_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) kref_init(&buf->kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) init_irq_work(&buf->wakeup_work, wakeup_readers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) irq_work_sync(&buf->wakeup_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) buf->subbufs_produced = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) buf->subbufs_consumed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) buf->bytes_consumed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) buf->finalized = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) buf->data = buf->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) buf->offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) for (i = 0; i < buf->chan->n_subbufs; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) buf->padding[i] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * relay_reset - reset the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * @chan: the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * This has the effect of erasing all data from all channel buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * and restarting the channel in its initial state. The buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * are not freed, so any mappings are still in effect.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * NOTE. Care should be taken that the channel isn't actually
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * being used by anything when this call is made.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) void relay_reset(struct rchan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) struct rchan_buf *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (!chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) __relay_reset(buf, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) mutex_lock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) for_each_possible_cpu(i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) if ((buf = *per_cpu_ptr(chan->buf, i)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) __relay_reset(buf, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) mutex_unlock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) EXPORT_SYMBOL_GPL(relay_reset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) static inline void relay_set_buf_dentry(struct rchan_buf *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) struct dentry *dentry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) buf->dentry = dentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) d_inode(buf->dentry)->i_size = buf->early_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) static struct dentry *relay_create_buf_file(struct rchan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) struct rchan_buf *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) struct dentry *dentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) char *tmpname;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) tmpname = kzalloc(NAME_MAX + 1, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (!tmpname)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) snprintf(tmpname, NAME_MAX, "%s%d", chan->base_filename, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) /* Create file in fs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) dentry = chan->cb->create_buf_file(tmpname, chan->parent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) S_IRUSR, buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) &chan->is_global);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) if (IS_ERR(dentry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) dentry = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) kfree(tmpname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) return dentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * relay_open_buf - create a new relay channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) * used by relay_open() and CPU hotplug.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) static struct rchan_buf *relay_open_buf(struct rchan *chan, unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) struct rchan_buf *buf = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) struct dentry *dentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) if (chan->is_global)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) return *per_cpu_ptr(chan->buf, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) buf = relay_create_buf(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) if (chan->has_base_filename) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) dentry = relay_create_buf_file(chan, buf, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (!dentry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) goto free_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) relay_set_buf_dentry(buf, dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) /* Only retrieve global info, nothing more, nothing less */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) dentry = chan->cb->create_buf_file(NULL, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) S_IRUSR, buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) &chan->is_global);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) if (IS_ERR_OR_NULL(dentry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) goto free_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) buf->cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) __relay_reset(buf, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if(chan->is_global) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) *per_cpu_ptr(chan->buf, 0) = buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) buf->cpu = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) return buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) free_buf:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) relay_destroy_buf(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * relay_close_buf - close a channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * @buf: channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * Marks the buffer finalized and restores the default callbacks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) * The channel buffer and channel buffer data structure are then freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * automatically when the last reference is given up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) static void relay_close_buf(struct rchan_buf *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) buf->finalized = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) irq_work_sync(&buf->wakeup_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) buf->chan->cb->remove_buf_file(buf->dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) kref_put(&buf->kref, relay_remove_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) static void setup_callbacks(struct rchan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) struct rchan_callbacks *cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (!cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) chan->cb = &default_channel_callbacks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) if (!cb->subbuf_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) cb->subbuf_start = subbuf_start_default_callback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (!cb->buf_mapped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) cb->buf_mapped = buf_mapped_default_callback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (!cb->buf_unmapped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) cb->buf_unmapped = buf_unmapped_default_callback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (!cb->create_buf_file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) cb->create_buf_file = create_buf_file_default_callback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) if (!cb->remove_buf_file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) cb->remove_buf_file = remove_buf_file_default_callback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) chan->cb = cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) int relay_prepare_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) struct rchan *chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) struct rchan_buf *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) mutex_lock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) list_for_each_entry(chan, &relay_channels, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if ((buf = *per_cpu_ptr(chan->buf, cpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) buf = relay_open_buf(chan, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) if (!buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) pr_err("relay: cpu %d buffer creation failed\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) mutex_unlock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) *per_cpu_ptr(chan->buf, cpu) = buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) mutex_unlock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) * relay_open - create a new relay channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * @base_filename: base name of files to create, %NULL for buffering only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) * @parent: dentry of parent directory, %NULL for root directory or buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * @subbuf_size: size of sub-buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * @n_subbufs: number of sub-buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * @cb: client callback functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) * @private_data: user-defined data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * Returns channel pointer if successful, %NULL otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * Creates a channel buffer for each cpu using the sizes and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * attributes specified. The created channel buffer files
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * will be named base_filename0...base_filenameN-1. File
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * permissions will be %S_IRUSR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * If opening a buffer (@parent = NULL) that you later wish to register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * in a filesystem, call relay_late_setup_files() once the @parent dentry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * is available.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct rchan *relay_open(const char *base_filename,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) struct dentry *parent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) size_t subbuf_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) size_t n_subbufs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) struct rchan_callbacks *cb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) void *private_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) struct rchan *chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) struct rchan_buf *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) if (!(subbuf_size && n_subbufs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) if (subbuf_size > UINT_MAX / n_subbufs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) chan = kzalloc(sizeof(struct rchan), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) if (!chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) chan->buf = alloc_percpu(struct rchan_buf *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (!chan->buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) kfree(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) chan->version = RELAYFS_CHANNEL_VERSION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) chan->n_subbufs = n_subbufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) chan->subbuf_size = subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) chan->alloc_size = PAGE_ALIGN(subbuf_size * n_subbufs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) chan->parent = parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) chan->private_data = private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) if (base_filename) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) chan->has_base_filename = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) strlcpy(chan->base_filename, base_filename, NAME_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) setup_callbacks(chan, cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) kref_init(&chan->kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) mutex_lock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) for_each_online_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) buf = relay_open_buf(chan, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) goto free_bufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) *per_cpu_ptr(chan->buf, i) = buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) list_add(&chan->list, &relay_channels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) mutex_unlock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) return chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) free_bufs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) for_each_possible_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) if ((buf = *per_cpu_ptr(chan->buf, i)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) relay_close_buf(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) kref_put(&chan->kref, relay_destroy_channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) mutex_unlock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) EXPORT_SYMBOL_GPL(relay_open);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) struct rchan_percpu_buf_dispatcher {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) struct rchan_buf *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) struct dentry *dentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) /* Called in atomic context. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) static void __relay_set_buf_dentry(void *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) struct rchan_percpu_buf_dispatcher *p = info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) relay_set_buf_dentry(p->buf, p->dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) * relay_late_setup_files - triggers file creation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) * @chan: channel to operate on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) * @base_filename: base name of files to create
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) * @parent: dentry of parent directory, %NULL for root directory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) * Returns 0 if successful, non-zero otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * Use to setup files for a previously buffer-only channel created
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) * by relay_open() with a NULL parent dentry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) * For example, this is useful for perfomring early tracing in kernel,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) * before VFS is up and then exposing the early results once the dentry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) * is available.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) int relay_late_setup_files(struct rchan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) const char *base_filename,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) struct dentry *parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) unsigned int i, curr_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) struct dentry *dentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) struct rchan_buf *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) struct rchan_percpu_buf_dispatcher disp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (!chan || !base_filename)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) strlcpy(chan->base_filename, base_filename, NAME_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) mutex_lock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) /* Is chan already set up? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) if (unlikely(chan->has_base_filename)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) mutex_unlock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) chan->has_base_filename = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) chan->parent = parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if (chan->is_global) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) buf = *per_cpu_ptr(chan->buf, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) if (!WARN_ON_ONCE(!buf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) dentry = relay_create_buf_file(chan, buf, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if (dentry && !WARN_ON_ONCE(!chan->is_global)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) relay_set_buf_dentry(buf, dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) mutex_unlock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) curr_cpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * The CPU hotplug notifier ran before us and created buffers with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * no files associated. So it's safe to call relay_setup_buf_file()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * on all currently online CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) for_each_online_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) buf = *per_cpu_ptr(chan->buf, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) if (unlikely(!buf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) WARN_ONCE(1, KERN_ERR "CPU has no buffer!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) dentry = relay_create_buf_file(chan, buf, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) if (unlikely(!dentry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) if (curr_cpu == i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) relay_set_buf_dentry(buf, dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) disp.buf = buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) disp.dentry = dentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) /* relay_channels_mutex must be held, so wait. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) err = smp_call_function_single(i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) __relay_set_buf_dentry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) &disp, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (unlikely(err))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) mutex_unlock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) EXPORT_SYMBOL_GPL(relay_late_setup_files);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) * relay_switch_subbuf - switch to a new sub-buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) * @buf: channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) * @length: size of current event
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * Returns either the length passed in or 0 if full.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * Performs sub-buffer-switch tasks such as invoking callbacks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * updating padding counts, waking up readers, etc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) void *old, *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) size_t old_subbuf, new_subbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) if (unlikely(length > buf->chan->subbuf_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) goto toobig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) if (buf->offset != buf->chan->subbuf_size + 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) buf->prev_padding = buf->chan->subbuf_size - buf->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) buf->padding[old_subbuf] = buf->prev_padding;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) buf->subbufs_produced++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) if (buf->dentry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) d_inode(buf->dentry)->i_size +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) buf->chan->subbuf_size -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) buf->padding[old_subbuf];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) buf->early_bytes += buf->chan->subbuf_size -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) buf->padding[old_subbuf];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (waitqueue_active(&buf->read_wait)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * Calling wake_up_interruptible() from here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) * will deadlock if we happen to be logging
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) * from the scheduler (trying to re-grab
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) * rq->lock), so defer it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) irq_work_queue(&buf->wakeup_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) old = buf->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) new = buf->start + new_subbuf * buf->chan->subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) buf->offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) buf->offset = buf->chan->subbuf_size + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) buf->data = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) buf->padding[new_subbuf] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) if (unlikely(length + buf->offset > buf->chan->subbuf_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) goto toobig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) return length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) toobig:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) buf->chan->last_toobig = length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) EXPORT_SYMBOL_GPL(relay_switch_subbuf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) * relay_subbufs_consumed - update the buffer's sub-buffers-consumed count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * @chan: the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) * @cpu: the cpu associated with the channel buffer to update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * @subbufs_consumed: number of sub-buffers to add to current buf's count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) * Adds to the channel buffer's consumed sub-buffer count.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) * subbufs_consumed should be the number of sub-buffers newly consumed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) * not the total consumed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) * NOTE. Kernel clients don't need to call this function if the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) * mode is 'overwrite'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) void relay_subbufs_consumed(struct rchan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) unsigned int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) size_t subbufs_consumed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) struct rchan_buf *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) if (!chan || cpu >= NR_CPUS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) buf = *per_cpu_ptr(chan->buf, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) if (!buf || subbufs_consumed > chan->n_subbufs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) buf->subbufs_consumed = buf->subbufs_produced;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) buf->subbufs_consumed += subbufs_consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) EXPORT_SYMBOL_GPL(relay_subbufs_consumed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) * relay_close - close the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * @chan: the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) * Closes all channel buffers and frees the channel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) void relay_close(struct rchan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) struct rchan_buf *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) if (!chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) mutex_lock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) relay_close_buf(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) for_each_possible_cpu(i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) if ((buf = *per_cpu_ptr(chan->buf, i)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) relay_close_buf(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) if (chan->last_toobig)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) printk(KERN_WARNING "relay: one or more items not logged "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) "[item size (%zd) > sub-buffer size (%zd)]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) chan->last_toobig, chan->subbuf_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) list_del(&chan->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) kref_put(&chan->kref, relay_destroy_channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) mutex_unlock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) EXPORT_SYMBOL_GPL(relay_close);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) * relay_flush - close the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * @chan: the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * Flushes all channel buffers, i.e. forces buffer switch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) void relay_flush(struct rchan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) struct rchan_buf *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) if (!chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) relay_switch_subbuf(buf, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) mutex_lock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) for_each_possible_cpu(i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) if ((buf = *per_cpu_ptr(chan->buf, i)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) relay_switch_subbuf(buf, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) mutex_unlock(&relay_channels_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) EXPORT_SYMBOL_GPL(relay_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) * relay_file_open - open file op for relay files
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * @inode: the inode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * @filp: the file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * Increments the channel buffer refcount.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) static int relay_file_open(struct inode *inode, struct file *filp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) struct rchan_buf *buf = inode->i_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) kref_get(&buf->kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) filp->private_data = buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) return nonseekable_open(inode, filp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) * relay_file_mmap - mmap file op for relay files
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) * @filp: the file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) * @vma: the vma describing what to map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * Calls upon relay_mmap_buf() to map the file into user space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) static int relay_file_mmap(struct file *filp, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) struct rchan_buf *buf = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) return relay_mmap_buf(buf, vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) * relay_file_poll - poll file op for relay files
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) * @filp: the file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) * @wait: poll table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) * Poll implemention.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) static __poll_t relay_file_poll(struct file *filp, poll_table *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) __poll_t mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) struct rchan_buf *buf = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) if (buf->finalized)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) return EPOLLERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) if (filp->f_mode & FMODE_READ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) poll_wait(filp, &buf->read_wait, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) if (!relay_buf_empty(buf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) mask |= EPOLLIN | EPOLLRDNORM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) return mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) * relay_file_release - release file op for relay files
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) * @inode: the inode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * @filp: the file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * Decrements the channel refcount, as the filesystem is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * no longer using it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) static int relay_file_release(struct inode *inode, struct file *filp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) struct rchan_buf *buf = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) kref_put(&buf->kref, relay_remove_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) * relay_file_read_consume - update the consumed count for the buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) static void relay_file_read_consume(struct rchan_buf *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) size_t read_pos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) size_t bytes_consumed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) size_t subbuf_size = buf->chan->subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) size_t n_subbufs = buf->chan->n_subbufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) size_t read_subbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) if (buf->subbufs_produced == buf->subbufs_consumed &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) buf->offset == buf->bytes_consumed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) if (buf->bytes_consumed + bytes_consumed > subbuf_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) relay_subbufs_consumed(buf->chan, buf->cpu, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) buf->bytes_consumed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) buf->bytes_consumed += bytes_consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) if (!read_pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) read_subbuf = buf->subbufs_consumed % n_subbufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) read_subbuf = read_pos / buf->chan->subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) if ((read_subbuf == buf->subbufs_produced % n_subbufs) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) (buf->offset == subbuf_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) relay_subbufs_consumed(buf->chan, buf->cpu, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) buf->bytes_consumed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) * relay_file_read_avail - boolean, are there unconsumed bytes available?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) static int relay_file_read_avail(struct rchan_buf *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) size_t subbuf_size = buf->chan->subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) size_t n_subbufs = buf->chan->n_subbufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) size_t produced = buf->subbufs_produced;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) size_t consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) relay_file_read_consume(buf, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) consumed = buf->subbufs_consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) if (unlikely(buf->offset > subbuf_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) if (produced == consumed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) if (unlikely(produced - consumed >= n_subbufs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) consumed = produced - n_subbufs + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) buf->subbufs_consumed = consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) buf->bytes_consumed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) produced = (produced % n_subbufs) * subbuf_size + buf->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) if (consumed > produced)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) produced += n_subbufs * subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) if (consumed == produced) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) if (buf->offset == subbuf_size &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) buf->subbufs_produced > buf->subbufs_consumed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * relay_file_read_subbuf_avail - return bytes available in sub-buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) * @read_pos: file read position
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) * @buf: relay channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) static size_t relay_file_read_subbuf_avail(size_t read_pos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) struct rchan_buf *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) size_t padding, avail = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) size_t read_subbuf, read_offset, write_subbuf, write_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) size_t subbuf_size = buf->chan->subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) write_subbuf = (buf->data - buf->start) / subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) read_subbuf = read_pos / subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) read_offset = read_pos % subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) padding = buf->padding[read_subbuf];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) if (read_subbuf == write_subbuf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) if (read_offset + padding < write_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) avail = write_offset - (read_offset + padding);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) avail = (subbuf_size - padding) - read_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) return avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) * relay_file_read_start_pos - find the first available byte to read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) * @buf: relay channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) * If the read_pos is in the middle of padding, return the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) * position of the first actually available byte, otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) * return the original value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) static size_t relay_file_read_start_pos(struct rchan_buf *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) size_t read_subbuf, padding, padding_start, padding_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) size_t subbuf_size = buf->chan->subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) size_t n_subbufs = buf->chan->n_subbufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) size_t consumed = buf->subbufs_consumed % n_subbufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) size_t read_pos = consumed * subbuf_size + buf->bytes_consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) read_subbuf = read_pos / subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) padding = buf->padding[read_subbuf];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) padding_start = (read_subbuf + 1) * subbuf_size - padding;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) padding_end = (read_subbuf + 1) * subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) if (read_pos >= padding_start && read_pos < padding_end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) read_subbuf = (read_subbuf + 1) % n_subbufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) read_pos = read_subbuf * subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) return read_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) * relay_file_read_end_pos - return the new read position
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) * @read_pos: file read position
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) * @buf: relay channel buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) * @count: number of bytes to be read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) static size_t relay_file_read_end_pos(struct rchan_buf *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) size_t read_pos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) size_t count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) size_t read_subbuf, padding, end_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) size_t subbuf_size = buf->chan->subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) size_t n_subbufs = buf->chan->n_subbufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) read_subbuf = read_pos / subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) padding = buf->padding[read_subbuf];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) if (read_pos % subbuf_size + count + padding == subbuf_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) end_pos = (read_subbuf + 1) * subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) end_pos = read_pos + count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) if (end_pos >= subbuf_size * n_subbufs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) end_pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) return end_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) static ssize_t relay_file_read(struct file *filp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) char __user *buffer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) size_t count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) struct rchan_buf *buf = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) size_t read_start, avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) size_t written = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) if (!count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) inode_lock(file_inode(filp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) void *from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) if (!relay_file_read_avail(buf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) read_start = relay_file_read_start_pos(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) avail = relay_file_read_subbuf_avail(read_start, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) if (!avail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) avail = min(count, avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) from = buf->start + read_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) ret = avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) if (copy_to_user(buffer, from, avail))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) buffer += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) written += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) count -= ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) relay_file_read_consume(buf, read_start, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) *ppos = relay_file_read_end_pos(buf, read_start, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) } while (count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) inode_unlock(file_inode(filp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) return written;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) static void relay_consume_bytes(struct rchan_buf *rbuf, int bytes_consumed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) rbuf->bytes_consumed += bytes_consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) if (rbuf->bytes_consumed >= rbuf->chan->subbuf_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) relay_subbufs_consumed(rbuf->chan, rbuf->cpu, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) rbuf->bytes_consumed %= rbuf->chan->subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) static void relay_pipe_buf_release(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) struct pipe_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) struct rchan_buf *rbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) rbuf = (struct rchan_buf *)page_private(buf->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) relay_consume_bytes(rbuf, buf->private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) static const struct pipe_buf_operations relay_pipe_buf_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) .release = relay_pipe_buf_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) .try_steal = generic_pipe_buf_try_steal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) .get = generic_pipe_buf_get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) static void relay_page_release(struct splice_pipe_desc *spd, unsigned int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) * subbuf_splice_actor - splice up to one subbuf's worth of data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) static ssize_t subbuf_splice_actor(struct file *in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) loff_t *ppos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) unsigned int flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) int *nonpad_ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) unsigned int pidx, poff, total_len, subbuf_pages, nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) struct rchan_buf *rbuf = in->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) unsigned int subbuf_size = rbuf->chan->subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) uint64_t pos = (uint64_t) *ppos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) uint32_t alloc_size = (uint32_t) rbuf->chan->alloc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) size_t read_start = (size_t) do_div(pos, alloc_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) size_t read_subbuf = read_start / subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) size_t padding = rbuf->padding[read_subbuf];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) size_t nonpad_end = read_subbuf * subbuf_size + subbuf_size - padding;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) struct page *pages[PIPE_DEF_BUFFERS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) struct partial_page partial[PIPE_DEF_BUFFERS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) struct splice_pipe_desc spd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) .pages = pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) .nr_pages = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) .nr_pages_max = PIPE_DEF_BUFFERS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) .partial = partial,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) .ops = &relay_pipe_buf_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) .spd_release = relay_page_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) ssize_t ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) if (rbuf->subbufs_produced == rbuf->subbufs_consumed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) if (splice_grow_spd(pipe, &spd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) * Adjust read len, if longer than what is available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) if (len > (subbuf_size - read_start % subbuf_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) len = subbuf_size - read_start % subbuf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) subbuf_pages = rbuf->chan->alloc_size >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) pidx = (read_start / PAGE_SIZE) % subbuf_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) poff = read_start & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) nr_pages = min_t(unsigned int, subbuf_pages, spd.nr_pages_max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) for (total_len = 0; spd.nr_pages < nr_pages; spd.nr_pages++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) unsigned int this_len, this_end, private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) unsigned int cur_pos = read_start + total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) if (!len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) this_len = min_t(unsigned long, len, PAGE_SIZE - poff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) private = this_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) spd.pages[spd.nr_pages] = rbuf->page_array[pidx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) spd.partial[spd.nr_pages].offset = poff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) this_end = cur_pos + this_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) if (this_end >= nonpad_end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) this_len = nonpad_end - cur_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) private = this_len + padding;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) spd.partial[spd.nr_pages].len = this_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) spd.partial[spd.nr_pages].private = private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) len -= this_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) total_len += this_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) poff = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) pidx = (pidx + 1) % subbuf_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) if (this_end >= nonpad_end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) spd.nr_pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) if (!spd.nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) ret = *nonpad_ret = splice_to_pipe(pipe, &spd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) if (ret < 0 || ret < total_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) if (read_start + ret == nonpad_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) ret += padding;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) splice_shrink_spd(&spd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) static ssize_t relay_file_splice_read(struct file *in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) loff_t *ppos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) ssize_t spliced;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) int nonpad_ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) spliced = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) while (len && !spliced) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) ret = subbuf_splice_actor(in, ppos, pipe, len, flags, &nonpad_ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) else if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) if (flags & SPLICE_F_NONBLOCK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) ret = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) *ppos += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) if (ret > len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) len -= ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) spliced += nonpad_ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) nonpad_ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) if (spliced)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) return spliced;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) const struct file_operations relay_file_operations = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) .open = relay_file_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) .poll = relay_file_poll,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) .mmap = relay_file_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) .read = relay_file_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) .llseek = no_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) .release = relay_file_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) .splice_read = relay_file_splice_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) EXPORT_SYMBOL_GPL(relay_file_operations);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags