^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/task_work.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/tracehook.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) static struct callback_head work_exited; /* all we need is ->next == NULL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * task_work_add - ask the @task to execute @work->func()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * @task: the task which should run the callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * @work: the callback to run
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * @notify: how to notify the targeted task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * Queue @work for task_work_run() below and notify the @task if @notify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * is @TWA_RESUME or @TWA_SIGNAL. @TWA_SIGNAL works like signals, in that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * it will interrupt the targeted task and run the task_work. @TWA_RESUME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * work is run only when the task exits the kernel and returns to user mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * or before entering guest mode. Fails if the @task is exiting/exited and thus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * it can't process this @work. Otherwise @work->func() will be called when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * @task goes through one of the aforementioned transitions, or exits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * If the targeted task is exiting, then an error is returned and the work item
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * is not queued. It's up to the caller to arrange for an alternative mechanism
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * in that case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * Note: there is no ordering guarantee on works queued here. The task_work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * list is LIFO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * RETURNS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * 0 if succeeds or -ESRCH.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) int task_work_add(struct task_struct *task, struct callback_head *work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) enum task_work_notify_mode notify)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) struct callback_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) /* record the work call stack in order to print it in KASAN reports */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) kasan_record_aux_stack(work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) head = READ_ONCE(task->task_works);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) if (unlikely(head == &work_exited))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) return -ESRCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) work->next = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) } while (cmpxchg(&task->task_works, head, work) != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) switch (notify) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) case TWA_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) case TWA_RESUME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) set_notify_resume(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) case TWA_SIGNAL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * Only grab the sighand lock if we don't already have some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * task_work pending. This pairs with the smp_store_mb()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * in get_signal(), see comment there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) if (!(READ_ONCE(task->jobctl) & JOBCTL_TASK_WORK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) lock_task_sighand(task, &flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) task->jobctl |= JOBCTL_TASK_WORK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) signal_wake_up(task, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) unlock_task_sighand(task, &flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * task_work_cancel - cancel a pending work added by task_work_add()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * @task: the task which should execute the work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * @func: identifies the work to remove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * Find the last queued pending work with ->func == @func and remove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * it from queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * RETURNS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * The found work or NULL if not found.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) struct callback_head *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) task_work_cancel(struct task_struct *task, task_work_func_t func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct callback_head **pprev = &task->task_works;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) struct callback_head *work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) if (likely(!task->task_works))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * If cmpxchg() fails we continue without updating pprev.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * Either we raced with task_work_add() which added the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * new entry before this work, we will find it again. Or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * we raced with task_work_run(), *pprev == NULL/exited.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) raw_spin_lock_irqsave(&task->pi_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) while ((work = READ_ONCE(*pprev))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (work->func != func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) pprev = &work->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) else if (cmpxchg(pprev, work, work->next) == work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) raw_spin_unlock_irqrestore(&task->pi_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * task_work_run - execute the works added by task_work_add()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * Flush the pending works. Should be used by the core kernel code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * Called before the task returns to the user-mode or stops, or when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * it exits. In the latter case task_work_add() can no longer add the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * new work after task_work_run() returns.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) void task_work_run(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct task_struct *task = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) struct callback_head *work, *head, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * work->func() can do task_work_add(), do not set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * work_exited unless the list is empty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) work = READ_ONCE(task->task_works);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) if (!work) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (task->flags & PF_EXITING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) head = &work_exited;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) } while (cmpxchg(&task->task_works, work, head) != work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (!work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * Synchronize with task_work_cancel(). It can not remove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * the first entry == work, cmpxchg(task_works) must fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * But it can remove another entry from the ->next list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) raw_spin_lock_irq(&task->pi_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) raw_spin_unlock_irq(&task->pi_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) next = work->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) work->func(work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) work = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) } while (work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags