^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * async.c: Asynchronous function calls for boot performance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * (C) Copyright 2009 Intel Corporation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Author: Arjan van de Ven <arjan@linux.intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) Goals and Theory of Operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) The primary goal of this feature is to reduce the kernel boot time,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) by doing various independent hardware delays and discovery operations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) decoupled and not strictly serialized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) More specifically, the asynchronous function call concept allows
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) certain operations (primarily during system boot) to happen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) asynchronously, out of order, while these operations still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) have their externally visible parts happen sequentially and in-order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) (not unlike how out-of-order CPUs retire their instructions in order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) Key to the asynchronous function call implementation is the concept of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) a "sequence cookie" (which, although it has an abstracted type, can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) thought of as a monotonically incrementing number).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) The async core will assign each scheduled event such a sequence cookie and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) pass this to the called functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) The asynchronously called function should before doing a globally visible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) operation, such as registering device numbers, call the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) async_synchronize_cookie() function and pass in its own cookie. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) async_synchronize_cookie() function will make sure that all asynchronous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) operations that were scheduled prior to the operation corresponding with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) cookie have completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) Subsystem/driver initialization code that scheduled asynchronous probe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) functions, but which shares global resources with other drivers/subsystems
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) that do not use the asynchronous call feature, need to do a full
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) synchronization with the async_synchronize_full() function, before returning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) from their init function. This is to maintain strict ordering between the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) asynchronous and synchronous parts of the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/async.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/ktime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/wait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include "workqueue_internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static async_cookie_t next_cookie = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define MAX_WORK 32768
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define ASYNC_COOKIE_MAX ULLONG_MAX /* infinity cookie */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) static LIST_HEAD(async_global_pending); /* pending from all registered doms */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static ASYNC_DOMAIN(async_dfl_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) static DEFINE_SPINLOCK(async_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct async_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct list_head domain_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct list_head global_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct work_struct work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) async_cookie_t cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) async_func_t func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) void *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct async_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static DECLARE_WAIT_QUEUE_HEAD(async_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static atomic_t entry_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static async_cookie_t lowest_in_progress(struct async_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct async_entry *first = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) async_cookie_t ret = ASYNC_COOKIE_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) spin_lock_irqsave(&async_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (domain) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (!list_empty(&domain->pending))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) first = list_first_entry(&domain->pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) struct async_entry, domain_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) if (!list_empty(&async_global_pending))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) first = list_first_entry(&async_global_pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) struct async_entry, global_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) if (first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) ret = first->cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) spin_unlock_irqrestore(&async_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * pick the first pending entry and run it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static void async_run_entry_fn(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct async_entry *entry =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) container_of(work, struct async_entry, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) ktime_t calltime, delta, rettime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /* 1) run (and print duration) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) if (initcall_debug && system_state < SYSTEM_RUNNING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) pr_debug("calling %lli_%pS @ %i\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) (long long)entry->cookie,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) entry->func, task_pid_nr(current));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) calltime = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) entry->func(entry->data, entry->cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (initcall_debug && system_state < SYSTEM_RUNNING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) rettime = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) delta = ktime_sub(rettime, calltime);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) pr_debug("initcall %lli_%pS returned 0 after %lld usecs\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) (long long)entry->cookie,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) entry->func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) (long long)ktime_to_ns(delta) >> 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /* 2) remove self from the pending queues */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) spin_lock_irqsave(&async_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) list_del_init(&entry->domain_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) list_del_init(&entry->global_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) /* 3) free the entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) kfree(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) atomic_dec(&entry_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) spin_unlock_irqrestore(&async_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) /* 4) wake up any waiters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) wake_up(&async_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * async_schedule_node_domain - NUMA specific version of async_schedule_domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * @func: function to execute asynchronously
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * @data: data pointer to pass to the function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * @node: NUMA node that we want to schedule this on or close to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * @domain: the domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * Returns an async_cookie_t that may be used for checkpointing later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * @domain may be used in the async_synchronize_*_domain() functions to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * wait within a certain synchronization domain rather than globally.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * Note: This function may be called from atomic or non-atomic contexts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * The node requested will be honored on a best effort basis. If the node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * has no CPUs associated with it then the work is distributed among all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * available CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) async_cookie_t async_schedule_node_domain(async_func_t func, void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) int node, struct async_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) struct async_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) async_cookie_t newcookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /* allow irq-off callers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * If we're out of memory or if there's too much work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * pending already, we execute synchronously.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) if (!entry || atomic_read(&entry_count) > MAX_WORK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) kfree(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) spin_lock_irqsave(&async_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) newcookie = next_cookie++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) spin_unlock_irqrestore(&async_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /* low on memory.. run synchronously */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) func(data, newcookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) return newcookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) INIT_LIST_HEAD(&entry->domain_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) INIT_LIST_HEAD(&entry->global_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) INIT_WORK(&entry->work, async_run_entry_fn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) entry->func = func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) entry->data = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) entry->domain = domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) spin_lock_irqsave(&async_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) /* allocate cookie and queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) newcookie = entry->cookie = next_cookie++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) list_add_tail(&entry->domain_list, &domain->pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) if (domain->registered)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) list_add_tail(&entry->global_list, &async_global_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) atomic_inc(&entry_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) spin_unlock_irqrestore(&async_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) /* schedule for execution */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) queue_work_node(node, system_unbound_wq, &entry->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) return newcookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) EXPORT_SYMBOL_GPL(async_schedule_node_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * async_schedule_node - NUMA specific version of async_schedule
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * @func: function to execute asynchronously
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * @data: data pointer to pass to the function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * @node: NUMA node that we want to schedule this on or close to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * Returns an async_cookie_t that may be used for checkpointing later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * Note: This function may be called from atomic or non-atomic contexts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * The node requested will be honored on a best effort basis. If the node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * has no CPUs associated with it then the work is distributed among all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * available CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) async_cookie_t async_schedule_node(async_func_t func, void *data, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) return async_schedule_node_domain(func, data, node, &async_dfl_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) EXPORT_SYMBOL_GPL(async_schedule_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * async_synchronize_full - synchronize all asynchronous function calls
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * This function waits until all asynchronous function calls have been done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) void async_synchronize_full(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) async_synchronize_full_domain(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) EXPORT_SYMBOL_GPL(async_synchronize_full);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * async_unregister_domain - ensure no more anonymous waiters on this domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * @domain: idle domain to flush out of any async_synchronize_full instances
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * async_synchronize_{cookie|full}_domain() are not flushed since callers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * of these routines should know the lifetime of @domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) void async_unregister_domain(struct async_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) spin_lock_irq(&async_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) WARN_ON(!domain->registered || !list_empty(&domain->pending));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) domain->registered = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) spin_unlock_irq(&async_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) EXPORT_SYMBOL_GPL(async_unregister_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * @domain: the domain to synchronize
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * This function waits until all asynchronous function calls for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * synchronization domain specified by @domain have been done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) void async_synchronize_full_domain(struct async_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * @cookie: async_cookie_t to use as checkpoint
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * @domain: the domain to synchronize (%NULL for all registered domains)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * This function waits until all asynchronous function calls for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * synchronization domain specified by @domain submitted prior to @cookie
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) * have been done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) ktime_t starttime, delta, endtime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) if (initcall_debug && system_state < SYSTEM_RUNNING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) pr_debug("async_waiting @ %i\n", task_pid_nr(current));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) starttime = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) wait_event(async_done, lowest_in_progress(domain) >= cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) if (initcall_debug && system_state < SYSTEM_RUNNING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) endtime = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) delta = ktime_sub(endtime, starttime);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) pr_debug("async_continuing @ %i after %lli usec\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) task_pid_nr(current),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) (long long)ktime_to_ns(delta) >> 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * @cookie: async_cookie_t to use as checkpoint
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * This function waits until all asynchronous function calls prior to @cookie
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) * have been done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) void async_synchronize_cookie(async_cookie_t cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) async_synchronize_cookie_domain(cookie, &async_dfl_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) EXPORT_SYMBOL_GPL(async_synchronize_cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) * current_is_async - is %current an async worker task?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * Returns %true if %current is an async worker task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) bool current_is_async(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) struct worker *worker = current_wq_worker();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) return worker && worker->current_func == async_run_entry_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) EXPORT_SYMBOL_GPL(current_is_async);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags