/*
*
* (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* A copy of the licence is included with the program, and can also be obtained
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
*/
#include <linux/anon_inodes.h>
#include <linux/atomic.h>
#include <linux/hrtimer.h>
#include <linux/jiffies.h>
#include <linux/kthread.h>
#include <linux/list.h>
#include <linux/mm.h>
#include <linux/poll.h>
#include <linux/preempt.h>
#include <linux/slab.h>
#include <linux/wait.h>
#include <mali_kbase.h>
#include <mali_kbase_hwaccess_instr.h>
#include <mali_kbase_hwaccess_jm.h>
#include <mali_kbase_hwcnt_reader.h>
#include <mali_kbase_mem_linux.h>
#include <mali_kbase_tlstream.h>
/*****************************************************************************/
/* Hwcnt reader API version */
#define HWCNT_READER_API 1
/* The number of nanoseconds in a second. */
#define NSECS_IN_SEC 1000000000ull /* ns */
/* The time resolution of dumping service. */
#define DUMPING_RESOLUTION 500000ull /* ns */
/* The maximal supported number of dumping buffers. */
#define MAX_BUFFER_COUNT 32
/* Size and number of hw counters blocks. */
#define NR_CNT_BLOCKS_PER_GROUP 8
#define NR_CNT_PER_BLOCK 64
#define NR_BYTES_PER_CNT 4
#define NR_BYTES_PER_HDR 16
#define PRFCNT_EN_MASK_OFFSET 0x8
/*****************************************************************************/
enum {
SHADER_HWCNT_BM,
TILER_HWCNT_BM,
MMU_L2_HWCNT_BM,
JM_HWCNT_BM
};
enum vinstr_state {
VINSTR_IDLE,
VINSTR_DUMPING,
VINSTR_SUSPENDING,
VINSTR_SUSPENDED,
VINSTR_RESUMING
};
/**
* struct kbase_vinstr_context - vinstr context per device
* @lock: protects the entire vinstr context
* @kbdev: pointer to kbase device
* @kctx: pointer to kbase context
* @vmap: vinstr vmap for mapping hwcnt dump buffer
* @gpu_va: GPU hwcnt dump buffer address
* @cpu_va: the CPU side mapping of the hwcnt dump buffer
* @dump_size: size of the dump buffer in bytes
* @bitmap: current set of counters monitored, not always in sync
* with hardware
* @reprogram: when true, reprogram hwcnt block with the new set of
* counters
* @state: vinstr state
* @state_lock: protects information about vinstr state
* @suspend_waitq: notification queue to trigger state re-validation
* @suspend_cnt: reference counter of vinstr's suspend state
* @suspend_work: worker to execute on entering suspended state
* @resume_work: worker to execute on leaving suspended state
* @nclients: number of attached clients, pending or otherwise
* @waiting_clients: head of list of clients being periodically sampled
* @idle_clients: head of list of clients being idle
* @suspended_clients: head of list of clients being suspended
* @thread: periodic sampling thread
* @waitq: notification queue of sampling thread
* @request_pending: request for action for sampling thread
*/
struct kbase_vinstr_context {
struct mutex lock;
struct kbase_device *kbdev;
struct kbase_context *kctx;
struct kbase_vmap_struct vmap;
u64 gpu_va;
void *cpu_va;
size_t dump_size;
u32 bitmap[4];
bool reprogram;
enum vinstr_state state;
struct spinlock state_lock;
wait_queue_head_t suspend_waitq;
unsigned int suspend_cnt;
struct work_struct suspend_work;
struct work_struct resume_work;
u32 nclients;
struct list_head waiting_clients;
struct list_head idle_clients;
struct list_head suspended_clients;
struct task_struct *thread;
wait_queue_head_t waitq;
atomic_t request_pending;
};
/**
* struct kbase_vinstr_client - a vinstr client attached to a vinstr context
* @vinstr_ctx: vinstr context client is attached to
* @list: node used to attach this client to list in vinstr context
* @buffer_count: number of buffers this client is using
* @event_mask: events this client reacts to
* @dump_size: size of one dump buffer in bytes
* @bitmap: bitmap request for JM, TILER, SHADER and MMU counters
* @legacy_buffer: userspace hwcnt dump buffer (legacy interface)
* @kernel_buffer: kernel hwcnt dump buffer (kernel client interface)
* @accum_buffer: temporary accumulation buffer for preserving counters
* @dump_time: next time this clients shall request hwcnt dump
* @dump_interval: interval between periodic hwcnt dumps
* @dump_buffers: kernel hwcnt dump buffers allocated by this client
* @dump_buffers_meta: metadata of dump buffers
* @meta_idx: index of metadata being accessed by userspace
* @read_idx: index of buffer read by userspace
* @write_idx: index of buffer being written by dumping service
* @waitq: client's notification queue
* @pending: when true, client has attached but hwcnt not yet updated
*/
struct kbase_vinstr_client {
struct kbase_vinstr_context *vinstr_ctx;
struct list_head list;
unsigned int buffer_count;
u32 event_mask;
size_t dump_size;
u32 bitmap[4];
void __user *legacy_buffer;
void *kernel_buffer;
void *accum_buffer;
u64 dump_time;
u32 dump_interval;
char *dump_buffers;
struct kbase_hwcnt_reader_metadata *dump_buffers_meta;
atomic_t meta_idx;
atomic_t read_idx;
atomic_t write_idx;
wait_queue_head_t waitq;
bool pending;
};
/**
* struct kbasep_vinstr_wake_up_timer - vinstr service thread wake up timer
* @hrtimer: high resolution timer
* @vinstr_ctx: vinstr context
*/
struct kbasep_vinstr_wake_up_timer {
struct hrtimer hrtimer;
struct kbase_vinstr_context *vinstr_ctx;
};
/*****************************************************************************/
static int kbasep_vinstr_service_task(void *data);
static unsigned int kbasep_vinstr_hwcnt_reader_poll(
struct file *filp,
poll_table *wait);
static long kbasep_vinstr_hwcnt_reader_ioctl(
struct file *filp,
unsigned int cmd,
unsigned long arg);
static int kbasep_vinstr_hwcnt_reader_mmap(
struct file *filp,
struct vm_area_struct *vma);
static int kbasep_vinstr_hwcnt_reader_release(
struct inode *inode,
struct file *filp);
/* The timeline stream file operations structure. */
static const struct file_operations vinstr_client_fops = {
.poll = kbasep_vinstr_hwcnt_reader_poll,
.unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
.compat_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
.mmap = kbasep_vinstr_hwcnt_reader_mmap,
.release = kbasep_vinstr_hwcnt_reader_release,
};
/*****************************************************************************/
static int enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
{
struct kbase_context *kctx = vinstr_ctx->kctx;
struct kbase_device *kbdev = kctx->kbdev;
struct kbase_uk_hwcnt_setup setup;
int err;
setup.dump_buffer = vinstr_ctx->gpu_va;
setup.jm_bm = vinstr_ctx->bitmap[JM_HWCNT_BM];
setup.tiler_bm = vinstr_ctx->bitmap[TILER_HWCNT_BM];
setup.shader_bm = vinstr_ctx->bitmap[SHADER_HWCNT_BM];
setup.mmu_l2_bm = vinstr_ctx->bitmap[MMU_L2_HWCNT_BM];
/* Mark the context as active so the GPU is kept turned on */
/* A suspend won't happen here, because we're in a syscall from a
* userspace thread. */
kbase_pm_context_active(kbdev);
/* Schedule the context in */
kbasep_js_schedule_privileged_ctx(kbdev, kctx);
err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &setup);
if (err) {
/* Release the context. This had its own Power Manager Active
* reference */
kbasep_js_release_privileged_ctx(kbdev, kctx);
/* Also release our Power Manager Active reference */
kbase_pm_context_idle(kbdev);
}
return err;
}
static void disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
{
struct kbase_context *kctx = vinstr_ctx->kctx;
struct kbase_device *kbdev = kctx->kbdev;
int err;
err = kbase_instr_hwcnt_disable_internal(kctx);
if (err) {
dev_warn(kbdev->dev, "Failed to disable HW counters (ctx:%p)",
kctx);
return;
}
/* Release the context. This had its own Power Manager Active reference. */
kbasep_js_release_privileged_ctx(kbdev, kctx);
/* Also release our Power Manager Active reference. */
kbase_pm_context_idle(kbdev);
dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx);
}
static int reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
{
disable_hwcnt(vinstr_ctx);
return enable_hwcnt(vinstr_ctx);
}
static void hwcnt_bitmap_set(u32 dst[4], u32 src[4])
{
dst[JM_HWCNT_BM] = src[JM_HWCNT_BM];
dst[TILER_HWCNT_BM] = src[TILER_HWCNT_BM];
dst[SHADER_HWCNT_BM] = src[SHADER_HWCNT_BM];
dst[MMU_L2_HWCNT_BM] = src[MMU_L2_HWCNT_BM];
}
static void hwcnt_bitmap_union(u32 dst[4], u32 src[4])
{
dst[JM_HWCNT_BM] |= src[JM_HWCNT_BM];
dst[TILER_HWCNT_BM] |= src[TILER_HWCNT_BM];
dst[SHADER_HWCNT_BM] |= src[SHADER_HWCNT_BM];
dst[MMU_L2_HWCNT_BM] |= src[MMU_L2_HWCNT_BM];
}
size_t kbase_vinstr_dump_size(struct kbase_device *kbdev)
{
size_t dump_size;
#ifndef CONFIG_MALI_NO_MALI
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) {
u32 nr_cg;
nr_cg = kbdev->gpu_props.num_core_groups;
dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP *
NR_CNT_PER_BLOCK *
NR_BYTES_PER_CNT;
} else
#endif /* CONFIG_MALI_NO_MALI */
{
/* assume v5 for now */
base_gpu_props *props = &kbdev->gpu_props.props;
u32 nr_l2 = props->l2_props.num_l2_slices;
u64 core_mask = props->coherency_info.group[0].core_mask;
u32 nr_blocks = fls64(core_mask);
/* JM and tiler counter blocks are always present */
dump_size = (2 + nr_l2 + nr_blocks) *
NR_CNT_PER_BLOCK *
NR_BYTES_PER_CNT;
}
return dump_size;
}
KBASE_EXPORT_TEST_API(kbase_vinstr_dump_size);
static size_t kbasep_vinstr_dump_size_ctx(
struct kbase_vinstr_context *vinstr_ctx)
{
return kbase_vinstr_dump_size(vinstr_ctx->kctx->kbdev);
}
static int kbasep_vinstr_map_kernel_dump_buffer(
struct kbase_vinstr_context *vinstr_ctx)
{
struct kbase_va_region *reg;
struct kbase_context *kctx = vinstr_ctx->kctx;
u64 flags, nr_pages;
flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR;
vinstr_ctx->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
nr_pages = PFN_UP(vinstr_ctx->dump_size);
reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags,
&vinstr_ctx->gpu_va);
if (!reg)
return -ENOMEM;
vinstr_ctx->cpu_va = kbase_vmap(
kctx,
vinstr_ctx->gpu_va,
vinstr_ctx->dump_size,
&vinstr_ctx->vmap);
if (!vinstr_ctx->cpu_va) {
kbase_mem_free(kctx, vinstr_ctx->gpu_va);
return -ENOMEM;
}
return 0;
}
static void kbasep_vinstr_unmap_kernel_dump_buffer(
struct kbase_vinstr_context *vinstr_ctx)
{
struct kbase_context *kctx = vinstr_ctx->kctx;
kbase_vunmap(kctx, &vinstr_ctx->vmap);
kbase_mem_free(kctx, vinstr_ctx->gpu_va);
}
/**
* kbasep_vinstr_create_kctx - create kernel context for vinstr
* @vinstr_ctx: vinstr context
* Return: zero on success
*/
static int kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx)
{
struct kbase_device *kbdev = vinstr_ctx->kbdev;
struct kbasep_kctx_list_element *element;
unsigned long flags;
bool enable_backend = false;
int err;
vinstr_ctx->kctx = kbase_create_context(vinstr_ctx->kbdev, true);
if (!vinstr_ctx->kctx)
return -ENOMEM;
/* Map the master kernel dump buffer. The HW dumps the counters
* into this memory region. */
err = kbasep_vinstr_map_kernel_dump_buffer(vinstr_ctx);
if (err) {
kbase_destroy_context(vinstr_ctx->kctx);
vinstr_ctx->kctx = NULL;
return err;
}
/* Add kernel context to list of contexts associated with device. */
element = kzalloc(sizeof(*element), GFP_KERNEL);
if (element) {
element->kctx = vinstr_ctx->kctx;
mutex_lock(&kbdev->kctx_list_lock);
list_add(&element->link, &kbdev->kctx_list);
/* Inform timeline client about new context.
* Do this while holding the lock to avoid tracepoint
* being created in both body and summary stream. */
KBASE_TLSTREAM_TL_NEW_CTX(
vinstr_ctx->kctx,
(u32)(vinstr_ctx->kctx->id),
(u32)(vinstr_ctx->kctx->tgid));
mutex_unlock(&kbdev->kctx_list_lock);
} else {
/* Don't treat this as a fail - just warn about it. */
dev_warn(kbdev->dev,
"couldn't add kctx to kctx_list\n");
}
/* Don't enable hardware counters if vinstr is suspended.
* Note that vinstr resume code is run under vinstr context lock,
* lower layer will be enabled as needed on resume. */
spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
if (VINSTR_IDLE == vinstr_ctx->state)
enable_backend = true;
spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
if (enable_backend)
err = enable_hwcnt(vinstr_ctx);
if (err) {
kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
kbase_destroy_context(vinstr_ctx->kctx);
if (element) {
mutex_lock(&kbdev->kctx_list_lock);
list_del(&element->link);
kfree(element);
mutex_unlock(&kbdev->kctx_list_lock);
}
KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx);
vinstr_ctx->kctx = NULL;
return err;
}
vinstr_ctx->thread = kthread_run(
kbasep_vinstr_service_task,
vinstr_ctx,
"mali_vinstr_service");
if (!vinstr_ctx->thread) {
disable_hwcnt(vinstr_ctx);
kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
kbase_destroy_context(vinstr_ctx->kctx);
if (element) {
mutex_lock(&kbdev->kctx_list_lock);
list_del(&element->link);
kfree(element);
mutex_unlock(&kbdev->kctx_list_lock);
}
KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx);
vinstr_ctx->kctx = NULL;
return -EFAULT;
}
return 0;
}
/**
* kbasep_vinstr_destroy_kctx - destroy vinstr's kernel context
* @vinstr_ctx: vinstr context
*/
static void kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx)
{
struct kbase_device *kbdev = vinstr_ctx->kbdev;
struct kbasep_kctx_list_element *element;
struct kbasep_kctx_list_element *tmp;
bool found = false;
/* Release hw counters dumping resources. */
vinstr_ctx->thread = NULL;
disable_hwcnt(vinstr_ctx);
kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
kbase_destroy_context(vinstr_ctx->kctx);
/* Remove kernel context from the device's contexts list. */
mutex_lock(&kbdev->kctx_list_lock);
list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) {
if (element->kctx == vinstr_ctx->kctx) {
list_del(&element->link);
kfree(element);
found = true;
}
}
mutex_unlock(&kbdev->kctx_list_lock);
if (!found)
dev_warn(kbdev->dev, "kctx not in kctx_list\n");
/* Inform timeline client about context destruction. */
KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx);
vinstr_ctx->kctx = NULL;
}
/**
* kbasep_vinstr_attach_client - Attach a client to the vinstr core
* @vinstr_ctx: vinstr context
* @buffer_count: requested number of dump buffers
* @bitmap: bitmaps describing which counters should be enabled
* @argp: pointer where notification descriptor shall be stored
* @kernel_buffer: pointer to kernel side buffer
*
* Return: vinstr opaque client handle or NULL on failure
*/
static struct kbase_vinstr_client *kbasep_vinstr_attach_client(
struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count,
u32 bitmap[4], void *argp, void *kernel_buffer)
{
struct task_struct *thread = NULL;
struct kbase_vinstr_client *cli;
KBASE_DEBUG_ASSERT(vinstr_ctx);
if (buffer_count > MAX_BUFFER_COUNT
|| (buffer_count & (buffer_count - 1)))
return NULL;
cli = kzalloc(sizeof(*cli), GFP_KERNEL);
if (!cli)
return NULL;
cli->vinstr_ctx = vinstr_ctx;
cli->buffer_count = buffer_count;
cli->event_mask =
(1 << BASE_HWCNT_READER_EVENT_MANUAL) |
(1 << BASE_HWCNT_READER_EVENT_PERIODIC);
cli->pending = true;
hwcnt_bitmap_set(cli->bitmap, bitmap);
mutex_lock(&vinstr_ctx->lock);
hwcnt_bitmap_union(vinstr_ctx->bitmap, cli->bitmap);
vinstr_ctx->reprogram = true;
/* If this is the first client, create the vinstr kbase
* context. This context is permanently resident until the
* last client exits. */
if (!vinstr_ctx->nclients) {
hwcnt_bitmap_set(vinstr_ctx->bitmap, cli->bitmap);
if (kbasep_vinstr_create_kctx(vinstr_ctx) < 0)
goto error;
vinstr_ctx->reprogram = false;
cli->pending = false;
}
/* The GPU resets the counter block every time there is a request
* to dump it. We need a per client kernel buffer for accumulating
* the counters. */
cli->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL);
if (!cli->accum_buffer)
goto error;
/* Prepare buffers. */
if (cli->buffer_count) {
int *fd = (int *)argp;
size_t tmp;
/* Allocate area for buffers metadata storage. */
tmp = sizeof(struct kbase_hwcnt_reader_metadata) *
cli->buffer_count;
cli->dump_buffers_meta = kmalloc(tmp, GFP_KERNEL);
if (!cli->dump_buffers_meta)
goto error;
/* Allocate required number of dumping buffers. */
cli->dump_buffers = (char *)__get_free_pages(
GFP_KERNEL | __GFP_ZERO,
get_order(cli->dump_size * cli->buffer_count));
if (!cli->dump_buffers)
goto error;
/* Create descriptor for user-kernel data exchange. */
*fd = anon_inode_getfd(
"[mali_vinstr_desc]",
&vinstr_client_fops,
cli,
O_RDONLY | O_CLOEXEC);
if (0 > *fd)
goto error;
} else if (kernel_buffer) {
cli->kernel_buffer = kernel_buffer;
} else {
cli->legacy_buffer = (void __user *)argp;
}
atomic_set(&cli->read_idx, 0);
atomic_set(&cli->meta_idx, 0);
atomic_set(&cli->write_idx, 0);
init_waitqueue_head(&cli->waitq);
vinstr_ctx->nclients++;
list_add(&cli->list, &vinstr_ctx->idle_clients);
mutex_unlock(&vinstr_ctx->lock);
return cli;
error:
kfree(cli->dump_buffers_meta);
if (cli->dump_buffers)
free_pages(
(unsigned long)cli->dump_buffers,
get_order(cli->dump_size * cli->buffer_count));
kfree(cli->accum_buffer);
if (!vinstr_ctx->nclients && vinstr_ctx->kctx) {
thread = vinstr_ctx->thread;
kbasep_vinstr_destroy_kctx(vinstr_ctx);
}
kfree(cli);
mutex_unlock(&vinstr_ctx->lock);
/* Thread must be stopped after lock is released. */
if (thread)
kthread_stop(thread);
return NULL;
}
void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli)
{
struct kbase_vinstr_context *vinstr_ctx;
struct kbase_vinstr_client *iter, *tmp;
struct task_struct *thread = NULL;
u32 zerobitmap[4] = { 0 };
int cli_found = 0;
KBASE_DEBUG_ASSERT(cli);
vinstr_ctx = cli->vinstr_ctx;
KBASE_DEBUG_ASSERT(vinstr_ctx);
mutex_lock(&vinstr_ctx->lock);
list_for_each_entry_safe(iter, tmp, &vinstr_ctx->idle_clients, list) {
if (iter == cli) {
vinstr_ctx->reprogram = true;
cli_found = 1;
list_del(&iter->list);
break;
}
}
if (!cli_found) {
list_for_each_entry_safe(
iter, tmp, &vinstr_ctx->waiting_clients, list) {
if (iter == cli) {
vinstr_ctx->reprogram = true;
cli_found = 1;
list_del(&iter->list);
break;
}
}
}
KBASE_DEBUG_ASSERT(cli_found);
kfree(cli->dump_buffers_meta);
free_pages(
(unsigned long)cli->dump_buffers,
get_order(cli->dump_size * cli->buffer_count));
kfree(cli->accum_buffer);
kfree(cli);
vinstr_ctx->nclients--;
if (!vinstr_ctx->nclients) {
thread = vinstr_ctx->thread;
kbasep_vinstr_destroy_kctx(vinstr_ctx);
}
/* Rebuild context bitmap now that the client has detached */
hwcnt_bitmap_set(vinstr_ctx->bitmap, zerobitmap);
list_for_each_entry(iter, &vinstr_ctx->idle_clients, list)
hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list)
hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
mutex_unlock(&vinstr_ctx->lock);
/* Thread must be stopped after lock is released. */
if (thread)
kthread_stop(thread);
}
KBASE_EXPORT_TEST_API(kbase_vinstr_detach_client);
/* Accumulate counters in the dump buffer */
static void accum_dump_buffer(void *dst, void *src, size_t dump_size)
{
size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
u32 *d = dst;
u32 *s = src;
size_t i, j;
for (i = 0; i < dump_size; i += block_size) {
/* skip over the header block */
d += NR_BYTES_PER_HDR / sizeof(u32);
s += NR_BYTES_PER_HDR / sizeof(u32);
for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) {
/* saturate result if addition would result in wraparound */
if (U32_MAX - *d < *s)
*d = U32_MAX;
else
*d += *s;
d++;
s++;
}
}
}
/* This is the Midgard v4 patch function. It copies the headers for each
* of the defined blocks from the master kernel buffer and then patches up
* the performance counter enable mask for each of the blocks to exclude
* counters that were not requested by the client. */
static void patch_dump_buffer_hdr_v4(
struct kbase_vinstr_context *vinstr_ctx,
struct kbase_vinstr_client *cli)
{
u32 *mask;
u8 *dst = cli->accum_buffer;
u8 *src = vinstr_ctx->cpu_va;
u32 nr_cg = vinstr_ctx->kctx->kbdev->gpu_props.num_core_groups;
size_t i, group_size, group;
enum {
SC0_BASE = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
SC1_BASE = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
SC2_BASE = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
SC3_BASE = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
TILER_BASE = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
JM_BASE = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT
};
group_size = NR_CNT_BLOCKS_PER_GROUP *
NR_CNT_PER_BLOCK *
NR_BYTES_PER_CNT;
for (i = 0; i < nr_cg; i++) {
group = i * group_size;
/* copy shader core headers */
memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE],
NR_BYTES_PER_HDR);
memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE],
NR_BYTES_PER_HDR);
memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE],
NR_BYTES_PER_HDR);
memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE],
NR_BYTES_PER_HDR);
/* copy tiler header */
memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE],
NR_BYTES_PER_HDR);
/* copy mmu header */
memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE],
NR_BYTES_PER_HDR);
/* copy job manager header */
memcpy(&dst[group + JM_BASE], &src[group + JM_BASE],
NR_BYTES_PER_HDR);
/* patch the shader core enable mask */
mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET];
*mask &= cli->bitmap[SHADER_HWCNT_BM];
mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET];
*mask &= cli->bitmap[SHADER_HWCNT_BM];
mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET];
*mask &= cli->bitmap[SHADER_HWCNT_BM];
mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET];
*mask &= cli->bitmap[SHADER_HWCNT_BM];
/* patch the tiler core enable mask */
mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET];
*mask &= cli->bitmap[TILER_HWCNT_BM];
/* patch the mmu core enable mask */
mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET];
*mask &= cli->bitmap[MMU_L2_HWCNT_BM];
/* patch the job manager enable mask */
mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET];
*mask &= cli->bitmap[JM_HWCNT_BM];
}
}
/* This is the Midgard v5 patch function. It copies the headers for each
* of the defined blocks from the master kernel buffer and then patches up
* the performance counter enable mask for each of the blocks to exclude
* counters that were not requested by the client. */
static void patch_dump_buffer_hdr_v5(
struct kbase_vinstr_context *vinstr_ctx,
struct kbase_vinstr_client *cli)
{
struct kbase_device *kbdev = vinstr_ctx->kctx->kbdev;
u32 i, nr_l2;
u64 core_mask;
u32 *mask;
u8 *dst = cli->accum_buffer;
u8 *src = vinstr_ctx->cpu_va;
size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
/* copy and patch job manager header */
memcpy(dst, src, NR_BYTES_PER_HDR);
mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
*mask &= cli->bitmap[JM_HWCNT_BM];
dst += block_size;
src += block_size;
/* copy and patch tiler header */
memcpy(dst, src, NR_BYTES_PER_HDR);
mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
*mask &= cli->bitmap[TILER_HWCNT_BM];
dst += block_size;
src += block_size;
/* copy and patch MMU/L2C headers */
nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices;
for (i = 0; i < nr_l2; i++) {
memcpy(dst, src, NR_BYTES_PER_HDR);
mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
*mask &= cli->bitmap[MMU_L2_HWCNT_BM];
dst += block_size;
src += block_size;
}
/* copy and patch shader core headers */
core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask;
while (0ull != core_mask) {
memcpy(dst, src, NR_BYTES_PER_HDR);
if (0ull != (core_mask & 1ull)) {
/* if block is not reserved update header */
mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
*mask &= cli->bitmap[SHADER_HWCNT_BM];
}
dst += block_size;
src += block_size;
core_mask >>= 1;
}
}
/**
* accum_clients - accumulate dumped hw counters for all known clients
* @vinstr_ctx: vinstr context
*/
static void accum_clients(struct kbase_vinstr_context *vinstr_ctx)
{
struct kbase_vinstr_client *iter;
int v4 = 0;
#ifndef CONFIG_MALI_NO_MALI
v4 = kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4);
#endif
list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) {
/* Don't bother accumulating clients whose hwcnt requests
* have not yet been honoured. */
if (iter->pending)
continue;
if (v4)
patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
else
patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
accum_dump_buffer(
iter->accum_buffer,
vinstr_ctx->cpu_va,
iter->dump_size);
}
list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) {
/* Don't bother accumulating clients whose hwcnt requests
* have not yet been honoured. */
if (iter->pending)
continue;
if (v4)
patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
else
patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
accum_dump_buffer(
iter->accum_buffer,
vinstr_ctx->cpu_va,
iter->dump_size);
}
}
/*****************************************************************************/
/**
* kbasep_vinstr_get_timestamp - return timestamp
*
* Function returns timestamp value based on raw monotonic timer. Value will
* wrap around zero in case of overflow.
*
* Return: timestamp value
*/
static u64 kbasep_vinstr_get_timestamp(void)
{
struct timespec64 ts;
ktime_get_raw_ts64(&ts);
return (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec;
}
/**
* kbasep_vinstr_add_dump_request - register client's dumping request
* @cli: requesting client
* @waiting_clients: list of pending dumping requests
*/
static void kbasep_vinstr_add_dump_request(
struct kbase_vinstr_client *cli,
struct list_head *waiting_clients)
{
struct kbase_vinstr_client *tmp;
if (list_empty(waiting_clients)) {
list_add(&cli->list, waiting_clients);
return;
}
list_for_each_entry(tmp, waiting_clients, list) {
if (tmp->dump_time > cli->dump_time) {
list_add_tail(&cli->list, &tmp->list);
return;
}
}
list_add_tail(&cli->list, waiting_clients);
}
/**
* kbasep_vinstr_collect_and_accumulate - collect hw counters via low level
* dump and accumulate them for known
* clients
* @vinstr_ctx: vinstr context
* @timestamp: pointer where collection timestamp will be recorded
*
* Return: zero on success
*/
static int kbasep_vinstr_collect_and_accumulate(
struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp)
{
unsigned long flags;
int rcode;
#ifdef CONFIG_MALI_NO_MALI
/* The dummy model needs the CPU mapping. */
gpu_model_set_dummy_prfcnt_base_cpu(vinstr_ctx->cpu_va);
#endif
spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
if (VINSTR_IDLE != vinstr_ctx->state) {
spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
return -EAGAIN;
} else {
vinstr_ctx->state = VINSTR_DUMPING;
}
spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
/* Request HW counters dump.
* Disable preemption to make dump timestamp more accurate. */
preempt_disable();
*timestamp = kbasep_vinstr_get_timestamp();
rcode = kbase_instr_hwcnt_request_dump(vinstr_ctx->kctx);
preempt_enable();
if (!rcode)
rcode = kbase_instr_hwcnt_wait_for_dump(vinstr_ctx->kctx);
WARN_ON(rcode);
spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
switch (vinstr_ctx->state)
{
case VINSTR_SUSPENDING:
schedule_work(&vinstr_ctx->suspend_work);
break;
case VINSTR_DUMPING:
vinstr_ctx->state = VINSTR_IDLE;
wake_up_all(&vinstr_ctx->suspend_waitq);
break;
default:
break;
}
spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
/* Accumulate values of collected counters. */
if (!rcode)
accum_clients(vinstr_ctx);
return rcode;
}
/**
* kbasep_vinstr_fill_dump_buffer - copy accumulated counters to empty kernel
* buffer
* @cli: requesting client
* @timestamp: timestamp when counters were collected
* @event_id: id of event that caused triggered counters collection
*
* Return: zero on success
*/
static int kbasep_vinstr_fill_dump_buffer(
struct kbase_vinstr_client *cli, u64 timestamp,
enum base_hwcnt_reader_event event_id)
{
unsigned int write_idx = atomic_read(&cli->write_idx);
unsigned int read_idx = atomic_read(&cli->read_idx);
struct kbase_hwcnt_reader_metadata *meta;
void *buffer;
/* Check if there is a place to copy HWC block into. */
if (write_idx - read_idx == cli->buffer_count)
return -1;
write_idx %= cli->buffer_count;
/* Fill in dump buffer and its metadata. */
buffer = &cli->dump_buffers[write_idx * cli->dump_size];
meta = &cli->dump_buffers_meta[write_idx];
meta->timestamp = timestamp;
meta->event_id = event_id;
meta->buffer_idx = write_idx;
memcpy(buffer, cli->accum_buffer, cli->dump_size);
return 0;
}
/**
* kbasep_vinstr_fill_dump_buffer_legacy - copy accumulated counters to buffer
* allocated in userspace
* @cli: requesting client
*
* Return: zero on success
*
* This is part of legacy ioctl interface.
*/
static int kbasep_vinstr_fill_dump_buffer_legacy(
struct kbase_vinstr_client *cli)
{
void __user *buffer = cli->legacy_buffer;
int rcode;
/* Copy data to user buffer. */
rcode = copy_to_user(buffer, cli->accum_buffer, cli->dump_size);
if (rcode)
pr_warn("error while copying buffer to user\n");
return rcode;
}
/**
* kbasep_vinstr_fill_dump_buffer_kernel - copy accumulated counters to buffer
* allocated in kernel space
* @cli: requesting client
*
* Return: zero on success
*
* This is part of the kernel client interface.
*/
static int kbasep_vinstr_fill_dump_buffer_kernel(
struct kbase_vinstr_client *cli)
{
memcpy(cli->kernel_buffer, cli->accum_buffer, cli->dump_size);
return 0;
}
/**
* kbasep_vinstr_reprogram - reprogram hwcnt set collected by inst
* @vinstr_ctx: vinstr context
*/
static void kbasep_vinstr_reprogram(
struct kbase_vinstr_context *vinstr_ctx)
{
unsigned long flags;
bool suspended = false;
/* Don't enable hardware counters if vinstr is suspended. */
spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
if (VINSTR_IDLE != vinstr_ctx->state)
suspended = true;
spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
if (suspended)
return;
/* Change to suspended state is done while holding vinstr context
* lock. Below code will then no re-enable the instrumentation. */
if (vinstr_ctx->reprogram) {
struct kbase_vinstr_client *iter;
if (!reprogram_hwcnt(vinstr_ctx)) {
vinstr_ctx->reprogram = false;
list_for_each_entry(
iter,
&vinstr_ctx->idle_clients,
list)
iter->pending = false;
list_for_each_entry(
iter,
&vinstr_ctx->waiting_clients,
list)
iter->pending = false;
}
}
}
/**
* kbasep_vinstr_update_client - copy accumulated counters to user readable
* buffer and notify the user
* @cli: requesting client
* @timestamp: timestamp when counters were collected
* @event_id: id of event that caused triggered counters collection
*
* Return: zero on success
*/
static int kbasep_vinstr_update_client(
struct kbase_vinstr_client *cli, u64 timestamp,
enum base_hwcnt_reader_event event_id)
{
int rcode = 0;
/* Copy collected counters to user readable buffer. */
if (cli->buffer_count)
rcode = kbasep_vinstr_fill_dump_buffer(
cli, timestamp, event_id);
else if (cli->kernel_buffer)
rcode = kbasep_vinstr_fill_dump_buffer_kernel(cli);
else
rcode = kbasep_vinstr_fill_dump_buffer_legacy(cli);
if (rcode)
goto exit;
/* Notify client. Make sure all changes to memory are visible. */
wmb();
atomic_inc(&cli->write_idx);
wake_up_interruptible(&cli->waitq);
/* Prepare for next request. */
memset(cli->accum_buffer, 0, cli->dump_size);
exit:
return rcode;
}
/**
* kbasep_vinstr_wake_up_callback - vinstr wake up timer wake up function
*
* @hrtimer: high resolution timer
*
* Return: High resolution timer restart enum.
*/
static enum hrtimer_restart kbasep_vinstr_wake_up_callback(
struct hrtimer *hrtimer)
{
struct kbasep_vinstr_wake_up_timer *timer =
container_of(
hrtimer,
struct kbasep_vinstr_wake_up_timer,
hrtimer);
KBASE_DEBUG_ASSERT(timer);
atomic_set(&timer->vinstr_ctx->request_pending, 1);
wake_up_all(&timer->vinstr_ctx->waitq);
return HRTIMER_NORESTART;
}
#ifdef CONFIG_DEBUG_OBJECT_TIMERS
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0))
/**
* kbase_destroy_hrtimer_on_stack - kernel's destroy_hrtimer_on_stack(),
* rewritten
*
* @timer: high resolution timer
*
* destroy_hrtimer_on_stack() was exported only for 4.7.0 kernel so for
* earlier kernel versions it is not possible to call it explicitly.
* Since this function must accompany hrtimer_init_on_stack(), which
* has to be used for hrtimer initialization if CONFIG_DEBUG_OBJECT_TIMERS
* is defined in order to avoid the warning about object on stack not being
* annotated, we rewrite it here to be used for earlier kernel versions.
*/
static void kbase_destroy_hrtimer_on_stack(struct hrtimer *timer)
{
debug_object_free(timer, &hrtimer_debug_descr);
}
#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) */
#endif /* CONFIG_DEBUG_OBJECT_TIMERS */
/**
* kbasep_vinstr_service_task - HWC dumping service thread
*
* @data: Pointer to vinstr context structure.
*
* Return: Always returns zero.
*/
static int kbasep_vinstr_service_task(void *data)
{
struct kbase_vinstr_context *vinstr_ctx = data;
struct kbasep_vinstr_wake_up_timer timer;
KBASE_DEBUG_ASSERT(vinstr_ctx);
hrtimer_init_on_stack(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
timer.hrtimer.function = kbasep_vinstr_wake_up_callback;
timer.vinstr_ctx = vinstr_ctx;
while (!kthread_should_stop()) {
struct kbase_vinstr_client *cli = NULL;
struct kbase_vinstr_client *tmp;
int rcode;
u64 timestamp = kbasep_vinstr_get_timestamp();
u64 dump_time = 0;
struct list_head expired_requests;
/* Hold lock while performing operations on lists of clients. */
mutex_lock(&vinstr_ctx->lock);
/* Closing thread must not interact with client requests. */
if (current == vinstr_ctx->thread) {
atomic_set(&vinstr_ctx->request_pending, 0);
if (!list_empty(&vinstr_ctx->waiting_clients)) {
cli = list_first_entry(
&vinstr_ctx->waiting_clients,
struct kbase_vinstr_client,
list);
dump_time = cli->dump_time;
}
}
if (!cli || ((s64)timestamp - (s64)dump_time < 0ll)) {
mutex_unlock(&vinstr_ctx->lock);
/* Sleep until next dumping event or service request. */
if (cli) {
u64 diff = dump_time - timestamp;
hrtimer_start(
&timer.hrtimer,
ns_to_ktime(diff),
HRTIMER_MODE_REL);
}
wait_event(
vinstr_ctx->waitq,
atomic_read(
&vinstr_ctx->request_pending) ||
kthread_should_stop());
hrtimer_cancel(&timer.hrtimer);
continue;
}
rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx,
×tamp);
INIT_LIST_HEAD(&expired_requests);
/* Find all expired requests. */
list_for_each_entry_safe(
cli,
tmp,
&vinstr_ctx->waiting_clients,
list) {
s64 tdiff =
(s64)(timestamp + DUMPING_RESOLUTION) -
(s64)cli->dump_time;
if (tdiff >= 0ll) {
list_del(&cli->list);
list_add(&cli->list, &expired_requests);
} else {
break;
}
}
/* Fill data for each request found. */
list_for_each_entry_safe(cli, tmp, &expired_requests, list) {
/* Ensure that legacy buffer will not be used from
* this kthread context. */
BUG_ON(0 == cli->buffer_count);
/* Expect only periodically sampled clients. */
BUG_ON(0 == cli->dump_interval);
if (!rcode)
kbasep_vinstr_update_client(
cli,
timestamp,
BASE_HWCNT_READER_EVENT_PERIODIC);
/* Set new dumping time. Drop missed probing times. */
do {
cli->dump_time += cli->dump_interval;
} while (cli->dump_time < timestamp);
list_del(&cli->list);
kbasep_vinstr_add_dump_request(
cli,
&vinstr_ctx->waiting_clients);
}
/* Reprogram counters set if required. */
kbasep_vinstr_reprogram(vinstr_ctx);
mutex_unlock(&vinstr_ctx->lock);
}
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0))
kbase_destroy_hrtimer_on_stack(&timer.hrtimer);
#else
destroy_hrtimer_on_stack(&timer.hrtimer);
#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) */
#endif /* CONFIG_DEBUG_OBJECTS_TIMERS */
return 0;
}
/*****************************************************************************/
/**
* kbasep_vinstr_hwcnt_reader_buffer_ready - check if client has ready buffers
* @cli: pointer to vinstr client structure
*
* Return: non-zero if client has at least one dumping buffer filled that was
* not notified to user yet
*/
static int kbasep_vinstr_hwcnt_reader_buffer_ready(
struct kbase_vinstr_client *cli)
{
KBASE_DEBUG_ASSERT(cli);
return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx);
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_get_buffer - hwcnt reader's ioctl command
* @cli: pointer to vinstr client structure
* @buffer: pointer to userspace buffer
* @size: size of buffer
*
* Return: zero on success
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
struct kbase_vinstr_client *cli, void __user *buffer,
size_t size)
{
unsigned int meta_idx = atomic_read(&cli->meta_idx);
unsigned int idx = meta_idx % cli->buffer_count;
struct kbase_hwcnt_reader_metadata *meta = &cli->dump_buffers_meta[idx];
/* Metadata sanity check. */
KBASE_DEBUG_ASSERT(idx == meta->buffer_idx);
if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
return -EINVAL;
/* Check if there is any buffer available. */
if (atomic_read(&cli->write_idx) == meta_idx)
return -EAGAIN;
/* Check if previously taken buffer was put back. */
if (atomic_read(&cli->read_idx) != meta_idx)
return -EBUSY;
/* Copy next available buffer's metadata to user. */
if (copy_to_user(buffer, meta, size))
return -EFAULT;
atomic_inc(&cli->meta_idx);
return 0;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_put_buffer - hwcnt reader's ioctl command
* @cli: pointer to vinstr client structure
* @buffer: pointer to userspace buffer
* @size: size of buffer
*
* Return: zero on success
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
struct kbase_vinstr_client *cli, void __user *buffer,
size_t size)
{
unsigned int read_idx = atomic_read(&cli->read_idx);
unsigned int idx = read_idx % cli->buffer_count;
struct kbase_hwcnt_reader_metadata meta;
if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
return -EINVAL;
/* Check if any buffer was taken. */
if (atomic_read(&cli->meta_idx) == read_idx)
return -EPERM;
/* Check if correct buffer is put back. */
if (copy_from_user(&meta, buffer, size))
return -EFAULT;
if (idx != meta.buffer_idx)
return -EINVAL;
atomic_inc(&cli->read_idx);
return 0;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_set_interval - hwcnt reader's ioctl command
* @cli: pointer to vinstr client structure
* @interval: periodic dumping interval (disable periodic dumping if zero)
*
* Return: zero on success
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
struct kbase_vinstr_client *cli, u32 interval)
{
struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
KBASE_DEBUG_ASSERT(vinstr_ctx);
mutex_lock(&vinstr_ctx->lock);
list_del(&cli->list);
cli->dump_interval = interval;
/* If interval is non-zero, enable periodic dumping for this client. */
if (cli->dump_interval) {
if (DUMPING_RESOLUTION > cli->dump_interval)
cli->dump_interval = DUMPING_RESOLUTION;
cli->dump_time =
kbasep_vinstr_get_timestamp() + cli->dump_interval;
kbasep_vinstr_add_dump_request(
cli, &vinstr_ctx->waiting_clients);
atomic_set(&vinstr_ctx->request_pending, 1);
wake_up_all(&vinstr_ctx->waitq);
} else {
list_add(&cli->list, &vinstr_ctx->idle_clients);
}
mutex_unlock(&vinstr_ctx->lock);
return 0;
}
/**
* kbasep_vinstr_hwcnt_reader_event_mask - return event mask for event id
* @event_id: id of event
* Return: event_mask or zero if event is not supported or maskable
*/
static u32 kbasep_vinstr_hwcnt_reader_event_mask(
enum base_hwcnt_reader_event event_id)
{
u32 event_mask = 0;
switch (event_id) {
case BASE_HWCNT_READER_EVENT_PREJOB:
case BASE_HWCNT_READER_EVENT_POSTJOB:
/* These event are maskable. */
event_mask = (1 << event_id);
break;
case BASE_HWCNT_READER_EVENT_MANUAL:
case BASE_HWCNT_READER_EVENT_PERIODIC:
/* These event are non-maskable. */
default:
/* These event are not supported. */
break;
}
return event_mask;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_enable_event - hwcnt reader's ioctl command
* @cli: pointer to vinstr client structure
* @event_id: id of event to enable
*
* Return: zero on success
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
struct kbase_vinstr_client *cli,
enum base_hwcnt_reader_event event_id)
{
struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
u32 event_mask;
KBASE_DEBUG_ASSERT(vinstr_ctx);
event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
if (!event_mask)
return -EINVAL;
mutex_lock(&vinstr_ctx->lock);
cli->event_mask |= event_mask;
mutex_unlock(&vinstr_ctx->lock);
return 0;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_disable_event - hwcnt reader's ioctl command
* @cli: pointer to vinstr client structure
* @event_id: id of event to disable
*
* Return: zero on success
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
struct kbase_vinstr_client *cli,
enum base_hwcnt_reader_event event_id)
{
struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
u32 event_mask;
KBASE_DEBUG_ASSERT(vinstr_ctx);
event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
if (!event_mask)
return -EINVAL;
mutex_lock(&vinstr_ctx->lock);
cli->event_mask &= ~event_mask;
mutex_unlock(&vinstr_ctx->lock);
return 0;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_get_hwver - hwcnt reader's ioctl command
* @cli: pointer to vinstr client structure
* @hwver: pointer to user buffer where hw version will be stored
*
* Return: zero on success
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
struct kbase_vinstr_client *cli, u32 __user *hwver)
{
#ifndef CONFIG_MALI_NO_MALI
struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
#endif
u32 ver = 5;
#ifndef CONFIG_MALI_NO_MALI
KBASE_DEBUG_ASSERT(vinstr_ctx);
if (kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4))
ver = 4;
#endif
return put_user(ver, hwver);
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl - hwcnt reader's ioctl
* @filp: pointer to file structure
* @cmd: user command
* @arg: command's argument
*
* Return: zero on success
*/
static long kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp,
unsigned int cmd, unsigned long arg)
{
long rcode = 0;
struct kbase_vinstr_client *cli;
KBASE_DEBUG_ASSERT(filp);
cli = filp->private_data;
KBASE_DEBUG_ASSERT(cli);
if (unlikely(KBASE_HWCNT_READER != _IOC_TYPE(cmd)))
return -EINVAL;
switch (cmd) {
case KBASE_HWCNT_READER_GET_API_VERSION:
rcode = put_user(HWCNT_READER_API, (u32 __user *)arg);
break;
case KBASE_HWCNT_READER_GET_HWVER:
rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
cli, (u32 __user *)arg);
break;
case KBASE_HWCNT_READER_GET_BUFFER_SIZE:
KBASE_DEBUG_ASSERT(cli->vinstr_ctx);
rcode = put_user(
(u32)cli->vinstr_ctx->dump_size,
(u32 __user *)arg);
break;
case KBASE_HWCNT_READER_DUMP:
rcode = kbase_vinstr_hwc_dump(
cli, BASE_HWCNT_READER_EVENT_MANUAL);
break;
case KBASE_HWCNT_READER_CLEAR:
rcode = kbase_vinstr_hwc_clear(cli);
break;
case KBASE_HWCNT_READER_GET_BUFFER:
rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
cli, (void __user *)arg, _IOC_SIZE(cmd));
break;
case KBASE_HWCNT_READER_PUT_BUFFER:
rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
cli, (void __user *)arg, _IOC_SIZE(cmd));
break;
case KBASE_HWCNT_READER_SET_INTERVAL:
rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
cli, (u32)arg);
break;
case KBASE_HWCNT_READER_ENABLE_EVENT:
rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
cli, (enum base_hwcnt_reader_event)arg);
break;
case KBASE_HWCNT_READER_DISABLE_EVENT:
rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
cli, (enum base_hwcnt_reader_event)arg);
break;
default:
rcode = -EINVAL;
break;
}
return rcode;
}
/**
* kbasep_vinstr_hwcnt_reader_poll - hwcnt reader's poll
* @filp: pointer to file structure
* @wait: pointer to poll table
* Return: POLLIN if data can be read without blocking, otherwise zero
*/
static unsigned int kbasep_vinstr_hwcnt_reader_poll(struct file *filp,
poll_table *wait)
{
struct kbase_vinstr_client *cli;
KBASE_DEBUG_ASSERT(filp);
KBASE_DEBUG_ASSERT(wait);
cli = filp->private_data;
KBASE_DEBUG_ASSERT(cli);
poll_wait(filp, &cli->waitq, wait);
if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli))
return POLLIN;
return 0;
}
/**
* kbasep_vinstr_hwcnt_reader_mmap - hwcnt reader's mmap
* @filp: pointer to file structure
* @vma: pointer to vma structure
* Return: zero on success
*/
static int kbasep_vinstr_hwcnt_reader_mmap(struct file *filp,
struct vm_area_struct *vma)
{
struct kbase_vinstr_client *cli;
unsigned long size, addr, pfn, offset;
unsigned long vm_size = vma->vm_end - vma->vm_start;
KBASE_DEBUG_ASSERT(filp);
KBASE_DEBUG_ASSERT(vma);
cli = filp->private_data;
KBASE_DEBUG_ASSERT(cli);
size = cli->buffer_count * cli->dump_size;
if (vma->vm_pgoff > (size >> PAGE_SHIFT))
return -EINVAL;
offset = vma->vm_pgoff << PAGE_SHIFT;
if (vm_size > size - offset)
return -EINVAL;
addr = __pa((unsigned long)cli->dump_buffers + offset);
pfn = addr >> PAGE_SHIFT;
return remap_pfn_range(
vma,
vma->vm_start,
pfn,
vm_size,
vma->vm_page_prot);
}
/**
* kbasep_vinstr_hwcnt_reader_release - hwcnt reader's release
* @inode: pointer to inode structure
* @filp: pointer to file structure
* Return always return zero
*/
static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode,
struct file *filp)
{
struct kbase_vinstr_client *cli;
KBASE_DEBUG_ASSERT(inode);
KBASE_DEBUG_ASSERT(filp);
cli = filp->private_data;
KBASE_DEBUG_ASSERT(cli);
kbase_vinstr_detach_client(cli);
return 0;
}
/*****************************************************************************/
/**
* kbasep_vinstr_kick_scheduler - trigger scheduler cycle
* @kbdev: pointer to kbase device structure
*/
static void kbasep_vinstr_kick_scheduler(struct kbase_device *kbdev)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
unsigned long flags;
down(&js_devdata->schedule_sem);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbase_backend_slot_update(kbdev);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
up(&js_devdata->schedule_sem);
}
/**
* kbasep_vinstr_suspend_worker - worker suspending vinstr module
* @data: pointer to work structure
*/
static void kbasep_vinstr_suspend_worker(struct work_struct *data)
{
struct kbase_vinstr_context *vinstr_ctx;
unsigned long flags;
vinstr_ctx = container_of(data, struct kbase_vinstr_context,
suspend_work);
mutex_lock(&vinstr_ctx->lock);
if (vinstr_ctx->kctx)
disable_hwcnt(vinstr_ctx);
spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
vinstr_ctx->state = VINSTR_SUSPENDED;
wake_up_all(&vinstr_ctx->suspend_waitq);
spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
mutex_unlock(&vinstr_ctx->lock);
/* Kick GPU scheduler to allow entering protected mode.
* This must happen after vinstr was suspended. */
kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev);
}
/**
* kbasep_vinstr_suspend_worker - worker resuming vinstr module
* @data: pointer to work structure
*/
static void kbasep_vinstr_resume_worker(struct work_struct *data)
{
struct kbase_vinstr_context *vinstr_ctx;
unsigned long flags;
vinstr_ctx = container_of(data, struct kbase_vinstr_context,
resume_work);
mutex_lock(&vinstr_ctx->lock);
if (vinstr_ctx->kctx)
enable_hwcnt(vinstr_ctx);
spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
vinstr_ctx->state = VINSTR_IDLE;
wake_up_all(&vinstr_ctx->suspend_waitq);
spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
mutex_unlock(&vinstr_ctx->lock);
/* Kick GPU scheduler to allow entering protected mode.
* Note that scheduler state machine might requested re-entry to
* protected mode before vinstr was resumed.
* This must happen after vinstr was release. */
kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev);
}
/*****************************************************************************/
struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev)
{
struct kbase_vinstr_context *vinstr_ctx;
vinstr_ctx = kzalloc(sizeof(*vinstr_ctx), GFP_KERNEL);
if (!vinstr_ctx)
return NULL;
INIT_LIST_HEAD(&vinstr_ctx->idle_clients);
INIT_LIST_HEAD(&vinstr_ctx->waiting_clients);
mutex_init(&vinstr_ctx->lock);
spin_lock_init(&vinstr_ctx->state_lock);
vinstr_ctx->kbdev = kbdev;
vinstr_ctx->thread = NULL;
vinstr_ctx->state = VINSTR_IDLE;
vinstr_ctx->suspend_cnt = 0;
INIT_WORK(&vinstr_ctx->suspend_work, kbasep_vinstr_suspend_worker);
INIT_WORK(&vinstr_ctx->resume_work, kbasep_vinstr_resume_worker);
init_waitqueue_head(&vinstr_ctx->suspend_waitq);
atomic_set(&vinstr_ctx->request_pending, 0);
init_waitqueue_head(&vinstr_ctx->waitq);
return vinstr_ctx;
}
void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx)
{
struct kbase_vinstr_client *cli;
/* Stop service thread first. */
if (vinstr_ctx->thread)
kthread_stop(vinstr_ctx->thread);
/* Wait for workers. */
flush_work(&vinstr_ctx->suspend_work);
flush_work(&vinstr_ctx->resume_work);
while (1) {
struct list_head *list = &vinstr_ctx->idle_clients;
if (list_empty(list)) {
list = &vinstr_ctx->waiting_clients;
if (list_empty(list))
break;
}
cli = list_first_entry(list, struct kbase_vinstr_client, list);
list_del(&cli->list);
kfree(cli->accum_buffer);
kfree(cli);
vinstr_ctx->nclients--;
}
KBASE_DEBUG_ASSERT(!vinstr_ctx->nclients);
if (vinstr_ctx->kctx)
kbasep_vinstr_destroy_kctx(vinstr_ctx);
kfree(vinstr_ctx);
}
int kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx,
struct kbase_uk_hwcnt_reader_setup *setup)
{
struct kbase_vinstr_client *cli;
u32 bitmap[4];
KBASE_DEBUG_ASSERT(vinstr_ctx);
KBASE_DEBUG_ASSERT(setup);
KBASE_DEBUG_ASSERT(setup->buffer_count);
bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
bitmap[TILER_HWCNT_BM] = setup->tiler_bm;
bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
bitmap[JM_HWCNT_BM] = setup->jm_bm;
cli = kbasep_vinstr_attach_client(
vinstr_ctx,
setup->buffer_count,
bitmap,
&setup->fd,
NULL);
if (!cli)
return -ENOMEM;
return 0;
}
int kbase_vinstr_legacy_hwc_setup(
struct kbase_vinstr_context *vinstr_ctx,
struct kbase_vinstr_client **cli,
struct kbase_uk_hwcnt_setup *setup)
{
KBASE_DEBUG_ASSERT(vinstr_ctx);
KBASE_DEBUG_ASSERT(setup);
KBASE_DEBUG_ASSERT(cli);
if (setup->dump_buffer) {
u32 bitmap[4];
bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
bitmap[TILER_HWCNT_BM] = setup->tiler_bm;
bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
bitmap[JM_HWCNT_BM] = setup->jm_bm;
if (*cli)
return -EBUSY;
*cli = kbasep_vinstr_attach_client(
vinstr_ctx,
0,
bitmap,
(void *)(long)setup->dump_buffer,
NULL);
if (!(*cli))
return -ENOMEM;
} else {
if (!*cli)
return -EINVAL;
kbase_vinstr_detach_client(*cli);
*cli = NULL;
}
return 0;
}
struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup(
struct kbase_vinstr_context *vinstr_ctx,
struct kbase_uk_hwcnt_reader_setup *setup,
void *kernel_buffer)
{
u32 bitmap[4];
if (!vinstr_ctx || !setup || !kernel_buffer)
return NULL;
bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
bitmap[TILER_HWCNT_BM] = setup->tiler_bm;
bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
bitmap[JM_HWCNT_BM] = setup->jm_bm;
return kbasep_vinstr_attach_client(
vinstr_ctx,
0,
bitmap,
NULL,
kernel_buffer);
}
KBASE_EXPORT_TEST_API(kbase_vinstr_hwcnt_kernel_setup);
int kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli,
enum base_hwcnt_reader_event event_id)
{
int rcode = 0;
struct kbase_vinstr_context *vinstr_ctx;
u64 timestamp;
u32 event_mask;
if (!cli)
return -EINVAL;
vinstr_ctx = cli->vinstr_ctx;
KBASE_DEBUG_ASSERT(vinstr_ctx);
KBASE_DEBUG_ASSERT(event_id < BASE_HWCNT_READER_EVENT_COUNT);
event_mask = 1 << event_id;
mutex_lock(&vinstr_ctx->lock);
if (event_mask & cli->event_mask) {
rcode = kbasep_vinstr_collect_and_accumulate(
vinstr_ctx,
×tamp);
if (rcode)
goto exit;
rcode = kbasep_vinstr_update_client(cli, timestamp, event_id);
if (rcode)
goto exit;
kbasep_vinstr_reprogram(vinstr_ctx);
}
exit:
mutex_unlock(&vinstr_ctx->lock);
return rcode;
}
KBASE_EXPORT_TEST_API(kbase_vinstr_hwc_dump);
int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli)
{
struct kbase_vinstr_context *vinstr_ctx;
int rcode;
u64 unused;
if (!cli)
return -EINVAL;
vinstr_ctx = cli->vinstr_ctx;
KBASE_DEBUG_ASSERT(vinstr_ctx);
mutex_lock(&vinstr_ctx->lock);
rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused);
if (rcode)
goto exit;
rcode = kbase_instr_hwcnt_clear(vinstr_ctx->kctx);
if (rcode)
goto exit;
memset(cli->accum_buffer, 0, cli->dump_size);
kbasep_vinstr_reprogram(vinstr_ctx);
exit:
mutex_unlock(&vinstr_ctx->lock);
return rcode;
}
int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx)
{
unsigned long flags;
int ret = -EAGAIN;
KBASE_DEBUG_ASSERT(vinstr_ctx);
spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
switch (vinstr_ctx->state) {
case VINSTR_SUSPENDED:
vinstr_ctx->suspend_cnt++;
/* overflow shall not happen */
BUG_ON(0 == vinstr_ctx->suspend_cnt);
ret = 0;
break;
case VINSTR_IDLE:
vinstr_ctx->state = VINSTR_SUSPENDING;
schedule_work(&vinstr_ctx->suspend_work);
break;
case VINSTR_DUMPING:
vinstr_ctx->state = VINSTR_SUSPENDING;
break;
case VINSTR_SUSPENDING:
/* fall through */
case VINSTR_RESUMING:
break;
default:
BUG();
break;
}
spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
return ret;
}
void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx)
{
wait_event(vinstr_ctx->suspend_waitq,
(0 == kbase_vinstr_try_suspend(vinstr_ctx)));
}
void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx)
{
unsigned long flags;
KBASE_DEBUG_ASSERT(vinstr_ctx);
spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
BUG_ON(VINSTR_SUSPENDING == vinstr_ctx->state);
if (VINSTR_SUSPENDED == vinstr_ctx->state) {
BUG_ON(0 == vinstr_ctx->suspend_cnt);
vinstr_ctx->suspend_cnt--;
if (0 == vinstr_ctx->suspend_cnt) {
vinstr_ctx->state = VINSTR_RESUMING;
schedule_work(&vinstr_ctx->resume_work);
}
}
spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
}
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags