^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Fence mechanism for dma-buf and to allow for asynchronous dma access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2012 Canonical Ltd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2012 Texas Instruments
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Authors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Rob Clark <robdclark@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Maarten Lankhorst <maarten.lankhorst@canonical.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/dma-fence.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #define CREATE_TRACE_POINTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <trace/events/dma_fence.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) EXPORT_TRACEPOINT_SYMBOL(dma_fence_emit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) EXPORT_TRACEPOINT_SYMBOL(dma_fence_enable_signal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) EXPORT_TRACEPOINT_SYMBOL(dma_fence_signaled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static DEFINE_SPINLOCK(dma_fence_stub_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static struct dma_fence dma_fence_stub;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * fence context counter: each execution context should have its own
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * fence context, this allows checking if fences belong to the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * context or not. One device can have multiple separate contexts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * and they're used if some engine can run independently of another.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) static atomic64_t dma_fence_context_counter = ATOMIC64_INIT(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * DOC: DMA fences overview
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * DMA fences, represented by &struct dma_fence, are the kernel internal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * synchronization primitive for DMA operations like GPU rendering, video
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * encoding/decoding, or displaying buffers on a screen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * A fence is initialized using dma_fence_init() and completed using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * dma_fence_signal(). Fences are associated with a context, allocated through
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * dma_fence_context_alloc(), and all fences on the same context are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * fully ordered.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * Since the purposes of fences is to facilitate cross-device and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * cross-application synchronization, there's multiple ways to use one:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * - Individual fences can be exposed as a &sync_file, accessed as a file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * descriptor from userspace, created by calling sync_file_create(). This is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * called explicit fencing, since userspace passes around explicit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * synchronization points.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * - Some subsystems also have their own explicit fencing primitives, like
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * &drm_syncobj. Compared to &sync_file, a &drm_syncobj allows the underlying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * fence to be updated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * - Then there's also implicit fencing, where the synchronization points are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * implicitly passed around as part of shared &dma_buf instances. Such
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * implicit fences are stored in &struct dma_resv through the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * &dma_buf.resv pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * DOC: fence cross-driver contract
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * Since &dma_fence provide a cross driver contract, all drivers must follow the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * same rules:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * * Fences must complete in a reasonable time. Fences which represent kernels
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * and shaders submitted by userspace, which could run forever, must be backed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * up by timeout and gpu hang recovery code. Minimally that code must prevent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * further command submission and force complete all in-flight fences, e.g.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * when the driver or hardware do not support gpu reset, or if the gpu reset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * failed for some reason. Ideally the driver supports gpu recovery which only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * affects the offending userspace context, and no other userspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * submissions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * * Drivers may have different ideas of what completion within a reasonable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * time means. Some hang recovery code uses a fixed timeout, others a mix
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * between observing forward progress and increasingly strict timeouts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * Drivers should not try to second guess timeout handling of fences from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * other drivers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * * To ensure there's no deadlocks of dma_fence_wait() against other locks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * drivers should annotate all code required to reach dma_fence_signal(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * which completes the fences, with dma_fence_begin_signalling() and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * dma_fence_end_signalling().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * * Drivers are allowed to call dma_fence_wait() while holding dma_resv_lock().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * This means any code required for fence completion cannot acquire a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * &dma_resv lock. Note that this also pulls in the entire established
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * locking hierarchy around dma_resv_lock() and dma_resv_unlock().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * * Drivers are allowed to call dma_fence_wait() from their &shrinker
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * callbacks. This means any code required for fence completion cannot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * allocate memory with GFP_KERNEL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * * Drivers are allowed to call dma_fence_wait() from their &mmu_notifier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * respectively &mmu_interval_notifier callbacks. This means any code required
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * for fence completeion cannot allocate memory with GFP_NOFS or GFP_NOIO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * Only GFP_ATOMIC is permissible, which might fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * Note that only GPU drivers have a reasonable excuse for both requiring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * &mmu_interval_notifier and &shrinker callbacks at the same time as having to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * track asynchronous compute work using &dma_fence. No driver outside of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * drivers/gpu should ever call dma_fence_wait() in such contexts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) static const char *dma_fence_stub_get_name(struct dma_fence *fence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return "stub";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) static const struct dma_fence_ops dma_fence_stub_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) .get_driver_name = dma_fence_stub_get_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) .get_timeline_name = dma_fence_stub_get_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * dma_fence_get_stub - return a signaled fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * Return a stub fence which is already signaled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) struct dma_fence *dma_fence_get_stub(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) spin_lock(&dma_fence_stub_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (!dma_fence_stub.ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) dma_fence_init(&dma_fence_stub,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) &dma_fence_stub_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) &dma_fence_stub_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) dma_fence_signal_locked(&dma_fence_stub);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) spin_unlock(&dma_fence_stub_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) return dma_fence_get(&dma_fence_stub);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) EXPORT_SYMBOL(dma_fence_get_stub);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * dma_fence_context_alloc - allocate an array of fence contexts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * @num: amount of contexts to allocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * This function will return the first index of the number of fence contexts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * allocated. The fence context is used for setting &dma_fence.context to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * unique number by passing the context to dma_fence_init().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) u64 dma_fence_context_alloc(unsigned num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) WARN_ON(!num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) return atomic64_fetch_add(num, &dma_fence_context_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) EXPORT_SYMBOL(dma_fence_context_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * DOC: fence signalling annotation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * Proving correctness of all the kernel code around &dma_fence through code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * review and testing is tricky for a few reasons:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * * It is a cross-driver contract, and therefore all drivers must follow the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * same rules for lock nesting order, calling contexts for various functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * and anything else significant for in-kernel interfaces. But it is also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * impossible to test all drivers in a single machine, hence brute-force N vs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * N testing of all combinations is impossible. Even just limiting to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * possible combinations is infeasible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * * There is an enormous amount of driver code involved. For render drivers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * there's the tail of command submission, after fences are published,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * scheduler code, interrupt and workers to process job completion,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * and timeout, gpu reset and gpu hang recovery code. Plus for integration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * with core mm with have &mmu_notifier, respectively &mmu_interval_notifier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * and &shrinker. For modesetting drivers there's the commit tail functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * between when fences for an atomic modeset are published, and when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * corresponding vblank completes, including any interrupt processing and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * related workers. Auditing all that code, across all drivers, is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * feasible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * * Due to how many other subsystems are involved and the locking hierarchies
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * this pulls in there is extremely thin wiggle-room for driver-specific
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * differences. &dma_fence interacts with almost all of the core memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * handling through page fault handlers via &dma_resv, dma_resv_lock() and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * dma_resv_unlock(). On the other side it also interacts through all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * allocation sites through &mmu_notifier and &shrinker.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * Furthermore lockdep does not handle cross-release dependencies, which means
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * any deadlocks between dma_fence_wait() and dma_fence_signal() can't be caught
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * at runtime with some quick testing. The simplest example is one thread
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * waiting on a &dma_fence while holding a lock::
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * lock(A);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * dma_fence_wait(B);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * unlock(A);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * while the other thread is stuck trying to acquire the same lock, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * prevents it from signalling the fence the previous thread is stuck waiting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * on::
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * lock(A);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * unlock(A);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * dma_fence_signal(B);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * By manually annotating all code relevant to signalling a &dma_fence we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * teach lockdep about these dependencies, which also helps with the validation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * headache since now lockdep can check all the rules for us::
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * cookie = dma_fence_begin_signalling();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * lock(A);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * unlock(A);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * dma_fence_signal(B);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * dma_fence_end_signalling(cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * For using dma_fence_begin_signalling() and dma_fence_end_signalling() to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * annotate critical sections the following rules need to be observed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * * All code necessary to complete a &dma_fence must be annotated, from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * point where a fence is accessible to other threads, to the point where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * dma_fence_signal() is called. Un-annotated code can contain deadlock issues,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * and due to the very strict rules and many corner cases it is infeasible to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * catch these just with review or normal stress testing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * * &struct dma_resv deserves a special note, since the readers are only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * protected by rcu. This means the signalling critical section starts as soon
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * as the new fences are installed, even before dma_resv_unlock() is called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * * The only exception are fast paths and opportunistic signalling code, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * calls dma_fence_signal() purely as an optimization, but is not required to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * guarantee completion of a &dma_fence. The usual example is a wait IOCTL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * which calls dma_fence_signal(), while the mandatory completion path goes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * through a hardware interrupt and possible job completion worker.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * * To aid composability of code, the annotations can be freely nested, as long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * as the overall locking hierarchy is consistent. The annotations also work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * both in interrupt and process context. Due to implementation details this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * requires that callers pass an opaque cookie from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * dma_fence_begin_signalling() to dma_fence_end_signalling().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * * Validation against the cross driver contract is implemented by priming
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * lockdep with the relevant hierarchy at boot-up. This means even just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * testing with a single device is enough to validate a driver, at least as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * far as deadlocks with dma_fence_wait() against dma_fence_signal() are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * concerned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) #ifdef CONFIG_LOCKDEP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) static struct lockdep_map dma_fence_lockdep_map = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) .name = "dma_fence_map"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * dma_fence_begin_signalling - begin a critical DMA fence signalling section
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) * Drivers should use this to annotate the beginning of any code section
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * required to eventually complete &dma_fence by calling dma_fence_signal().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * The end of these critical sections are annotated with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * dma_fence_end_signalling().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * Opaque cookie needed by the implementation, which needs to be passed to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * dma_fence_end_signalling().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) bool dma_fence_begin_signalling(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) /* explicitly nesting ... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (lock_is_held_type(&dma_fence_lockdep_map, 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) /* rely on might_sleep check for soft/hardirq locks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (in_atomic())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) /* ... and non-recursive readlock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) lock_acquire(&dma_fence_lockdep_map, 0, 0, 1, 1, NULL, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) EXPORT_SYMBOL(dma_fence_begin_signalling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * dma_fence_end_signalling - end a critical DMA fence signalling section
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * @cookie: opaque cookie from dma_fence_begin_signalling()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * Closes a critical section annotation opened by dma_fence_begin_signalling().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) void dma_fence_end_signalling(bool cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) lock_release(&dma_fence_lockdep_map, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) EXPORT_SYMBOL(dma_fence_end_signalling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) void __dma_fence_might_wait(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) bool tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) tmp = lock_is_held_type(&dma_fence_lockdep_map, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) lock_release(&dma_fence_lockdep_map, _THIS_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) lock_map_acquire(&dma_fence_lockdep_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) lock_map_release(&dma_fence_lockdep_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) lock_acquire(&dma_fence_lockdep_map, 0, 0, 1, 1, NULL, _THIS_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * dma_fence_signal_timestamp_locked - signal completion of a fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * @fence: the fence to signal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) * Signal completion for software callbacks on a fence, this will unblock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * dma_fence_wait() calls and run all the callbacks added with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) * dma_fence_add_callback(). Can be called multiple times, but since a fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * can only go from the unsignaled to the signaled state and not back, it will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * only be effective the first time. Set the timestamp provided as the fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * signal timestamp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * Unlike dma_fence_signal_timestamp(), this function must be called with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * &dma_fence.lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * Returns 0 on success and a negative error value when @fence has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * signalled already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) int dma_fence_signal_timestamp_locked(struct dma_fence *fence,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) ktime_t timestamp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) struct dma_fence_cb *cur, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) struct list_head cb_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) lockdep_assert_held(fence->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (unlikely(test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) &fence->flags)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) /* Stash the cb_list before replacing it with the timestamp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) list_replace(&fence->cb_list, &cb_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) fence->timestamp = timestamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) trace_dma_fence_signaled(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) list_for_each_entry_safe(cur, tmp, &cb_list, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) INIT_LIST_HEAD(&cur->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) cur->func(fence, cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) EXPORT_SYMBOL(dma_fence_signal_timestamp_locked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) * dma_fence_signal_timestamp - signal completion of a fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * @fence: the fence to signal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) * Signal completion for software callbacks on a fence, this will unblock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * dma_fence_wait() calls and run all the callbacks added with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * dma_fence_add_callback(). Can be called multiple times, but since a fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * can only go from the unsignaled to the signaled state and not back, it will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * only be effective the first time. Set the timestamp provided as the fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * signal timestamp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * Returns 0 on success and a negative error value when @fence has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * signalled already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (!fence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) spin_lock_irqsave(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) ret = dma_fence_signal_timestamp_locked(fence, timestamp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) spin_unlock_irqrestore(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) EXPORT_SYMBOL(dma_fence_signal_timestamp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * dma_fence_signal_locked - signal completion of a fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) * @fence: the fence to signal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) * Signal completion for software callbacks on a fence, this will unblock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) * dma_fence_wait() calls and run all the callbacks added with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) * dma_fence_add_callback(). Can be called multiple times, but since a fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) * can only go from the unsignaled to the signaled state and not back, it will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * only be effective the first time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) * held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * Returns 0 on success and a negative error value when @fence has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * signalled already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) int dma_fence_signal_locked(struct dma_fence *fence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) return dma_fence_signal_timestamp_locked(fence, ktime_get());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) EXPORT_SYMBOL(dma_fence_signal_locked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * dma_fence_signal - signal completion of a fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * @fence: the fence to signal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * Signal completion for software callbacks on a fence, this will unblock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * dma_fence_wait() calls and run all the callbacks added with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * dma_fence_add_callback(). Can be called multiple times, but since a fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) * can only go from the unsignaled to the signaled state and not back, it will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * only be effective the first time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * Returns 0 on success and a negative error value when @fence has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * signalled already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) int dma_fence_signal(struct dma_fence *fence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) bool tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) if (!fence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) tmp = dma_fence_begin_signalling();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) spin_lock_irqsave(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) ret = dma_fence_signal_timestamp_locked(fence, ktime_get());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) spin_unlock_irqrestore(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) dma_fence_end_signalling(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) EXPORT_SYMBOL(dma_fence_signal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * dma_fence_wait_timeout - sleep until the fence gets signaled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * or until timeout elapses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * @fence: the fence to wait on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * @intr: if true, do an interruptible wait
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * remaining timeout in jiffies on success. Other error values may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * returned on custom implementations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * Performs a synchronous wait on this fence. It is assumed the caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * directly or indirectly (buf-mgr between reservation and committing)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * holds a reference to the fence, otherwise the fence might be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * freed before return, resulting in undefined behavior.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * See also dma_fence_wait() and dma_fence_wait_any_timeout().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) signed long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) signed long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (WARN_ON(timeout < 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) __dma_fence_might_wait();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) trace_dma_fence_wait_start(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (fence->ops->wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) ret = fence->ops->wait(fence, intr, timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) ret = dma_fence_default_wait(fence, intr, timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) trace_dma_fence_wait_end(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) EXPORT_SYMBOL(dma_fence_wait_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * dma_fence_release - default relese function for fences
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) * @kref: &dma_fence.recfount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) * This is the default release functions for &dma_fence. Drivers shouldn't call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * this directly, but instead call dma_fence_put().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) void dma_fence_release(struct kref *kref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) struct dma_fence *fence =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) container_of(kref, struct dma_fence, refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) trace_dma_fence_destroy(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (WARN(!list_empty(&fence->cb_list) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) "Fence %s:%s:%llx:%llx released with pending signals!\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) fence->ops->get_driver_name(fence),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) fence->ops->get_timeline_name(fence),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) fence->context, fence->seqno)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) * Failed to signal before release, likely a refcounting issue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * This should never happen, but if it does make sure that we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * don't leave chains dangling. We set the error flag first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * so that the callbacks know this signal is due to an error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) spin_lock_irqsave(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) fence->error = -EDEADLK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) dma_fence_signal_locked(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) spin_unlock_irqrestore(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (fence->ops->release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) fence->ops->release(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) dma_fence_free(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) EXPORT_SYMBOL(dma_fence_release);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * dma_fence_free - default release function for &dma_fence.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * @fence: fence to release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) * This is the default implementation for &dma_fence_ops.release. It calls
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) * kfree_rcu() on @fence.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) void dma_fence_free(struct dma_fence *fence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) kfree_rcu(fence, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) EXPORT_SYMBOL(dma_fence_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) static bool __dma_fence_enable_signaling(struct dma_fence *fence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) bool was_set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) lockdep_assert_held(fence->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) &fence->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) if (!was_set && fence->ops->enable_signaling) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) trace_dma_fence_enable_signal(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) if (!fence->ops->enable_signaling(fence)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) dma_fence_signal_locked(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * dma_fence_enable_sw_signaling - enable signaling on fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * @fence: the fence to enable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * This will request for sw signaling to be enabled, to make the fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) * complete as soon as possible. This calls &dma_fence_ops.enable_signaling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) * internally.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) void dma_fence_enable_sw_signaling(struct dma_fence *fence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) spin_lock_irqsave(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) __dma_fence_enable_signaling(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) spin_unlock_irqrestore(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) EXPORT_SYMBOL(dma_fence_enable_sw_signaling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) * dma_fence_add_callback - add a callback to be called when the fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) * is signaled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) * @fence: the fence to wait on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * @cb: the callback to register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * @func: the function to call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * @cb will be initialized by dma_fence_add_callback(), no initialization
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * by the caller is required. Any number of callbacks can be registered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) * to a fence, but a callback can only be registered to one fence at a time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) * Note that the callback can be called from an atomic context. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) * fence is already signaled, this function will return -ENOENT (and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) * *not* call the callback).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) * Add a software callback to the fence. Same restrictions apply to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) * refcount as it does to dma_fence_wait(), however the caller doesn't need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) * keep a refcount to fence afterward dma_fence_add_callback() has returned:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) * when software access is enabled, the creator of the fence is required to keep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) * the fence alive until after it signals with dma_fence_signal(). The callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) * itself can be called from irq context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) * Returns 0 in case of success, -ENOENT if the fence is already signaled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) * and -EINVAL in case of error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) dma_fence_func_t func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (WARN_ON(!fence || !func))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) INIT_LIST_HEAD(&cb->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) spin_lock_irqsave(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (__dma_fence_enable_signaling(fence)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) cb->func = func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) list_add_tail(&cb->node, &fence->cb_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) INIT_LIST_HEAD(&cb->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) ret = -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) spin_unlock_irqrestore(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) EXPORT_SYMBOL(dma_fence_add_callback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) * dma_fence_get_status - returns the status upon completion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) * @fence: the dma_fence to query
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) * This wraps dma_fence_get_status_locked() to return the error status
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) * condition on a signaled fence. See dma_fence_get_status_locked() for more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) * Returns 0 if the fence has not yet been signaled, 1 if the fence has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) * been signaled without an error condition, or a negative error code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) * if the fence has been completed in err.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) int dma_fence_get_status(struct dma_fence *fence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) int status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) spin_lock_irqsave(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) status = dma_fence_get_status_locked(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) spin_unlock_irqrestore(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) return status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) EXPORT_SYMBOL(dma_fence_get_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * dma_fence_remove_callback - remove a callback from the signaling list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) * @fence: the fence to wait on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * @cb: the callback to remove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * Remove a previously queued callback from the fence. This function returns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * true if the callback is successfully removed, or false if the fence has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * already been signaled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * *WARNING*:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * Cancelling a callback should only be done if you really know what you're
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * doing, since deadlocks and race conditions could occur all too easily. For
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) * this reason, it should only ever be done on hardware lockup recovery,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) * with a reference held to the fence.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) * Behaviour is undefined if @cb has not been added to @fence using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) * dma_fence_add_callback() beforehand.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) bool ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) spin_lock_irqsave(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) ret = !list_empty(&cb->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) list_del_init(&cb->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) spin_unlock_irqrestore(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) EXPORT_SYMBOL(dma_fence_remove_callback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) struct default_wait_cb {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) struct dma_fence_cb base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) struct task_struct *task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) struct default_wait_cb *wait =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) container_of(cb, struct default_wait_cb, base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) wake_up_state(wait->task, TASK_NORMAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) * dma_fence_default_wait - default sleep until the fence gets signaled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) * or until timeout elapses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) * @fence: the fence to wait on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) * @intr: if true, do an interruptible wait
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) * remaining timeout in jiffies on success. If timeout is zero the value one is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) * returned if the fence is already signaled for consistency with other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) * functions taking a jiffies timeout.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) signed long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) struct default_wait_cb cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) signed long ret = timeout ? timeout : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) spin_lock_irqsave(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) if (intr && signal_pending(current)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) ret = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (!__dma_fence_enable_signaling(fence))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) if (!timeout) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) cb.base.func = dma_fence_default_wait_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) cb.task = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) list_add(&cb.base.node, &fence->cb_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) if (intr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) __set_current_state(TASK_INTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) __set_current_state(TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) spin_unlock_irqrestore(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) ret = schedule_timeout(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) spin_lock_irqsave(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (ret > 0 && intr && signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) ret = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) if (!list_empty(&cb.base.node))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) list_del(&cb.base.node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) __set_current_state(TASK_RUNNING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) spin_unlock_irqrestore(fence->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) EXPORT_SYMBOL(dma_fence_default_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) uint32_t *idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) for (i = 0; i < count; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) struct dma_fence *fence = fences[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) if (idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) *idx = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * dma_fence_wait_any_timeout - sleep until any fence gets signaled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) * or until timeout elapses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) * @fences: array of fences to wait on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * @count: number of fences to wait on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) * @intr: if true, do an interruptible wait
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) * @idx: used to store the first signaled fence index, meaningful only on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) * positive return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) * interrupted, 0 if the wait timed out, or the remaining timeout in jiffies
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) * on success.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * Synchronous waits for the first fence in the array to be signaled. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * caller needs to hold a reference to all fences in the array, otherwise a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * fence might be freed before return, resulting in undefined behavior.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) * See also dma_fence_wait() and dma_fence_wait_timeout().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) signed long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) bool intr, signed long timeout, uint32_t *idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) struct default_wait_cb *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) signed long ret = timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) unsigned i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) if (WARN_ON(!fences || !count || timeout < 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) if (timeout == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) for (i = 0; i < count; ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) if (dma_fence_is_signaled(fences[i])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) if (idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) *idx = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) cb = kcalloc(count, sizeof(struct default_wait_cb), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) if (cb == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) goto err_free_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) for (i = 0; i < count; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) struct dma_fence *fence = fences[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) cb[i].task = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) if (dma_fence_add_callback(fence, &cb[i].base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) dma_fence_default_wait_cb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) /* This fence is already signaled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) if (idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) *idx = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) goto fence_rm_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) while (ret > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) if (intr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) set_current_state(TASK_INTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) set_current_state(TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) if (dma_fence_test_signaled_any(fences, count, idx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) ret = schedule_timeout(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) if (ret > 0 && intr && signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) ret = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) __set_current_state(TASK_RUNNING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) fence_rm_cb:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) while (i-- > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) dma_fence_remove_callback(fences[i], &cb[i].base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) err_free_cb:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) kfree(cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) EXPORT_SYMBOL(dma_fence_wait_any_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) * dma_fence_init - Initialize a custom fence.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) * @fence: the fence to initialize
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) * @ops: the dma_fence_ops for operations on this fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) * @lock: the irqsafe spinlock to use for locking this fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * @context: the execution context this fence is run on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * @seqno: a linear increasing sequence number for this context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * Initializes an allocated fence, the caller doesn't have to keep its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * refcount after committing with this fence, but it will need to hold a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * refcount again if &dma_fence_ops.enable_signaling gets called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) * context and seqno are used for easy comparison between fences, allowing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) * to check which fence is later by simply using dma_fence_later().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) spinlock_t *lock, u64 context, u64 seqno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) BUG_ON(!lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) kref_init(&fence->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) fence->ops = ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) INIT_LIST_HEAD(&fence->cb_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) fence->lock = lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) fence->context = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) fence->seqno = seqno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) fence->flags = 0UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) fence->error = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) trace_dma_fence_init(fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) EXPORT_SYMBOL(dma_fence_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags