// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2011-2022 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*/
#include "mali_kbase_vinstr.h"
#include "mali_kbase_hwcnt_virtualizer.h"
#include "mali_kbase_hwcnt_types.h"
#include <uapi/gpu/arm/bifrost/mali_kbase_hwcnt_reader.h>
#include "mali_kbase_hwcnt_gpu.h"
#include "mali_kbase_hwcnt_gpu_narrow.h"
#include <uapi/gpu/arm/bifrost/mali_kbase_ioctl.h>
#include "mali_malisw.h"
#include "mali_kbase_debug.h"
#include <linux/anon_inodes.h>
#include <linux/fcntl.h>
#include <linux/fs.h>
#include <linux/hrtimer.h>
#include <linux/log2.h>
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/poll.h>
#include <linux/slab.h>
#include <linux/version_compat_defs.h>
#include <linux/workqueue.h>
/* Hwcnt reader API version */
#define HWCNT_READER_API 1
/* The minimum allowed interval between dumps (equivalent to 10KHz) */
#define DUMP_INTERVAL_MIN_NS (100 * NSEC_PER_USEC)
/* The maximum allowed buffers per client */
#define MAX_BUFFER_COUNT 32
/**
* struct kbase_vinstr_context - IOCTL interface for userspace hardware
* counters.
* @hvirt: Hardware counter virtualizer used by vinstr.
* @metadata: Hardware counter metadata provided by virtualizer.
* @metadata_user: API compatible hardware counter metadata provided by vinstr.
* For compatibility with the user driver interface, this
* contains a narrowed version of the HWCNT metadata limited
* to 64 entries per block of 32 bits each.
* @lock: Lock protecting all vinstr state.
* @suspend_count: Suspend reference count. If non-zero, timer and worker are
* prevented from being re-scheduled.
* @client_count: Number of vinstr clients.
* @clients: List of vinstr clients.
* @dump_timer: Timer that enqueues dump_work to a workqueue.
* @dump_work: Worker for performing periodic counter dumps.
*/
struct kbase_vinstr_context {
struct kbase_hwcnt_virtualizer *hvirt;
const struct kbase_hwcnt_metadata *metadata;
const struct kbase_hwcnt_metadata_narrow *metadata_user;
struct mutex lock;
size_t suspend_count;
size_t client_count;
struct list_head clients;
struct hrtimer dump_timer;
struct work_struct dump_work;
};
/**
* struct kbase_vinstr_client - A vinstr client attached to a vinstr context.
* @vctx: Vinstr context client is attached to.
* @hvcli: Hardware counter virtualizer client.
* @node: Node used to attach this client to list in vinstr
* context.
* @dump_interval_ns: Interval between periodic dumps. If 0, not a periodic
* client.
* @next_dump_time_ns: Time in ns when this client's next periodic dump must
* occur. If 0, not a periodic client.
* @enable_map: Counters enable map.
* @tmp_buf: Temporary buffer to use before handing dump to client.
* @dump_bufs: Array of narrow dump buffers allocated by this client.
* @dump_bufs_meta: Metadata of hwcnt reader client buffers.
* @meta_idx: Index of metadata being accessed by userspace.
* @read_idx: Index of buffer read by userspace.
* @write_idx: Index of buffer being written by dump worker.
* @waitq: Client's notification queue.
*/
struct kbase_vinstr_client {
struct kbase_vinstr_context *vctx;
struct kbase_hwcnt_virtualizer_client *hvcli;
struct list_head node;
u64 next_dump_time_ns;
u32 dump_interval_ns;
struct kbase_hwcnt_enable_map enable_map;
struct kbase_hwcnt_dump_buffer tmp_buf;
struct kbase_hwcnt_dump_buffer_narrow_array dump_bufs;
struct kbase_hwcnt_reader_metadata *dump_bufs_meta;
atomic_t meta_idx;
atomic_t read_idx;
atomic_t write_idx;
wait_queue_head_t waitq;
};
static __poll_t kbasep_vinstr_hwcnt_reader_poll(struct file *filp, poll_table *wait);
static long kbasep_vinstr_hwcnt_reader_ioctl(
struct file *filp,
unsigned int cmd,
unsigned long arg);
static int kbasep_vinstr_hwcnt_reader_mmap(
struct file *filp,
struct vm_area_struct *vma);
static int kbasep_vinstr_hwcnt_reader_release(
struct inode *inode,
struct file *filp);
/* Vinstr client file operations */
static const struct file_operations vinstr_client_fops = {
.owner = THIS_MODULE,
.poll = kbasep_vinstr_hwcnt_reader_poll,
.unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
.compat_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
.mmap = kbasep_vinstr_hwcnt_reader_mmap,
.release = kbasep_vinstr_hwcnt_reader_release,
};
/**
* kbasep_vinstr_timestamp_ns() - Get the current time in nanoseconds.
*
* Return: Current time in nanoseconds.
*/
static u64 kbasep_vinstr_timestamp_ns(void)
{
return ktime_get_raw_ns();
}
/**
* kbasep_vinstr_next_dump_time_ns() - Calculate the next periodic dump time.
* @cur_ts_ns: Current time in nanoseconds.
* @interval: Interval between dumps in nanoseconds.
*
* Return: 0 if interval is 0 (i.e. a non-periodic client), or the next dump
* time that occurs after cur_ts_ns.
*/
static u64 kbasep_vinstr_next_dump_time_ns(u64 cur_ts_ns, u32 interval)
{
/* Non-periodic client */
if (interval == 0)
return 0;
/*
* Return the next interval after the current time relative to t=0.
* This means multiple clients with the same period will synchronise,
* regardless of when they were started, allowing the worker to be
* scheduled less frequently.
*/
do_div(cur_ts_ns, interval);
return (cur_ts_ns + 1) * interval;
}
/**
* kbasep_vinstr_client_dump() - Perform a dump for a client.
* @vcli: Non-NULL pointer to a vinstr client.
* @event_id: Event type that triggered the dump.
*
* Return: 0 on success, else error code.
*/
static int kbasep_vinstr_client_dump(
struct kbase_vinstr_client *vcli,
enum base_hwcnt_reader_event event_id)
{
int errcode;
u64 ts_start_ns;
u64 ts_end_ns;
unsigned int write_idx;
unsigned int read_idx;
struct kbase_hwcnt_dump_buffer *tmp_buf;
struct kbase_hwcnt_dump_buffer_narrow *dump_buf;
struct kbase_hwcnt_reader_metadata *meta;
u8 clk_cnt;
WARN_ON(!vcli);
lockdep_assert_held(&vcli->vctx->lock);
write_idx = atomic_read(&vcli->write_idx);
read_idx = atomic_read(&vcli->read_idx);
/* Check if there is a place to copy HWC block into. */
if (write_idx - read_idx == vcli->dump_bufs.buf_cnt)
return -EBUSY;
write_idx %= vcli->dump_bufs.buf_cnt;
dump_buf = &vcli->dump_bufs.bufs[write_idx];
meta = &vcli->dump_bufs_meta[write_idx];
tmp_buf = &vcli->tmp_buf;
errcode = kbase_hwcnt_virtualizer_client_dump(
vcli->hvcli, &ts_start_ns, &ts_end_ns, tmp_buf);
if (errcode)
return errcode;
/* Patch the dump buf headers, to hide the counters that other hwcnt
* clients are using.
*/
kbase_hwcnt_gpu_patch_dump_headers(tmp_buf, &vcli->enable_map);
/* Copy the temp buffer to the userspace visible buffer. The strict
* variant will explicitly zero any non-enabled counters to ensure
* nothing except exactly what the user asked for is made visible.
*
* A narrow copy is required since virtualizer has a bigger buffer
* but user only needs part of it.
*/
kbase_hwcnt_dump_buffer_copy_strict_narrow(dump_buf, tmp_buf,
&vcli->enable_map);
clk_cnt = vcli->vctx->metadata->clk_cnt;
meta->timestamp = ts_end_ns;
meta->event_id = event_id;
meta->buffer_idx = write_idx;
meta->cycles.top = (clk_cnt > 0) ? dump_buf->clk_cnt_buf[0] : 0;
meta->cycles.shader_cores =
(clk_cnt > 1) ? dump_buf->clk_cnt_buf[1] : 0;
/* Notify client. Make sure all changes to memory are visible. */
wmb();
atomic_inc(&vcli->write_idx);
wake_up_interruptible(&vcli->waitq);
return 0;
}
/**
* kbasep_vinstr_client_clear() - Reset all the client's counters to zero.
* @vcli: Non-NULL pointer to a vinstr client.
*
* Return: 0 on success, else error code.
*/
static int kbasep_vinstr_client_clear(struct kbase_vinstr_client *vcli)
{
u64 ts_start_ns;
u64 ts_end_ns;
WARN_ON(!vcli);
lockdep_assert_held(&vcli->vctx->lock);
/* A virtualizer dump with a NULL buffer will just clear the virtualizer
* client's buffer.
*/
return kbase_hwcnt_virtualizer_client_dump(
vcli->hvcli, &ts_start_ns, &ts_end_ns, NULL);
}
/**
* kbasep_vinstr_reschedule_worker() - Update next dump times for all periodic
* vinstr clients, then reschedule the dump
* worker appropriately.
* @vctx: Non-NULL pointer to the vinstr context.
*
* If there are no periodic clients, then the dump worker will not be
* rescheduled. Else, the dump worker will be rescheduled for the next periodic
* client dump.
*/
static void kbasep_vinstr_reschedule_worker(struct kbase_vinstr_context *vctx)
{
u64 cur_ts_ns;
u64 earliest_next_ns = U64_MAX;
struct kbase_vinstr_client *pos;
WARN_ON(!vctx);
lockdep_assert_held(&vctx->lock);
cur_ts_ns = kbasep_vinstr_timestamp_ns();
/*
* Update each client's next dump time, and find the earliest next
* dump time if any of the clients have a non-zero interval.
*/
list_for_each_entry(pos, &vctx->clients, node) {
const u64 cli_next_ns =
kbasep_vinstr_next_dump_time_ns(
cur_ts_ns, pos->dump_interval_ns);
/* Non-zero next dump time implies a periodic client */
if ((cli_next_ns != 0) && (cli_next_ns < earliest_next_ns))
earliest_next_ns = cli_next_ns;
pos->next_dump_time_ns = cli_next_ns;
}
/* Cancel the timer if it is already pending */
hrtimer_cancel(&vctx->dump_timer);
/* Start the timer if there are periodic clients and vinstr is not
* suspended.
*/
if ((earliest_next_ns != U64_MAX) &&
(vctx->suspend_count == 0) &&
!WARN_ON(earliest_next_ns < cur_ts_ns))
hrtimer_start(
&vctx->dump_timer,
ns_to_ktime(earliest_next_ns - cur_ts_ns),
HRTIMER_MODE_REL);
}
/**
* kbasep_vinstr_dump_worker()- Dump worker, that dumps all periodic clients
* that need to be dumped, then reschedules itself.
* @work: Work structure.
*/
static void kbasep_vinstr_dump_worker(struct work_struct *work)
{
struct kbase_vinstr_context *vctx =
container_of(work, struct kbase_vinstr_context, dump_work);
struct kbase_vinstr_client *pos;
u64 cur_time_ns;
mutex_lock(&vctx->lock);
cur_time_ns = kbasep_vinstr_timestamp_ns();
/* Dump all periodic clients whose next dump time is before the current
* time.
*/
list_for_each_entry(pos, &vctx->clients, node) {
if ((pos->next_dump_time_ns != 0) &&
(pos->next_dump_time_ns < cur_time_ns))
kbasep_vinstr_client_dump(
pos, BASE_HWCNT_READER_EVENT_PERIODIC);
}
/* Update the next dump times of all periodic clients, then reschedule
* this worker at the earliest next dump time.
*/
kbasep_vinstr_reschedule_worker(vctx);
mutex_unlock(&vctx->lock);
}
/**
* kbasep_vinstr_dump_timer() - Dump timer that schedules the dump worker for
* execution as soon as possible.
* @timer: Timer structure.
*
* Return: HRTIMER_NORESTART always.
*/
static enum hrtimer_restart kbasep_vinstr_dump_timer(struct hrtimer *timer)
{
struct kbase_vinstr_context *vctx =
container_of(timer, struct kbase_vinstr_context, dump_timer);
/* We don't need to check vctx->suspend_count here, as the suspend
* function will ensure that any worker enqueued here is immediately
* cancelled, and the worker itself won't reschedule this timer if
* suspend_count != 0.
*/
kbase_hwcnt_virtualizer_queue_work(vctx->hvirt, &vctx->dump_work);
return HRTIMER_NORESTART;
}
/**
* kbasep_vinstr_client_destroy() - Destroy a vinstr client.
* @vcli: vinstr client. Must not be attached to a vinstr context.
*/
static void kbasep_vinstr_client_destroy(struct kbase_vinstr_client *vcli)
{
if (!vcli)
return;
kbase_hwcnt_virtualizer_client_destroy(vcli->hvcli);
kfree(vcli->dump_bufs_meta);
kbase_hwcnt_dump_buffer_narrow_array_free(&vcli->dump_bufs);
kbase_hwcnt_dump_buffer_free(&vcli->tmp_buf);
kbase_hwcnt_enable_map_free(&vcli->enable_map);
kfree(vcli);
}
/**
* kbasep_vinstr_client_create() - Create a vinstr client. Does not attach to
* the vinstr context.
* @vctx: Non-NULL pointer to vinstr context.
* @setup: Non-NULL pointer to hardware counter ioctl setup structure.
* setup->buffer_count must not be 0 and must be a power of 2.
* @out_vcli: Non-NULL pointer to where created client will be stored on
* success.
*
* Return: 0 on success, else error code.
*/
static int kbasep_vinstr_client_create(
struct kbase_vinstr_context *vctx,
struct kbase_ioctl_hwcnt_reader_setup *setup,
struct kbase_vinstr_client **out_vcli)
{
int errcode;
struct kbase_vinstr_client *vcli;
struct kbase_hwcnt_physical_enable_map phys_em;
WARN_ON(!vctx);
WARN_ON(!setup);
WARN_ON(setup->buffer_count == 0);
WARN_ON(!is_power_of_2(setup->buffer_count));
vcli = kzalloc(sizeof(*vcli), GFP_KERNEL);
if (!vcli)
return -ENOMEM;
vcli->vctx = vctx;
errcode = kbase_hwcnt_enable_map_alloc(
vctx->metadata, &vcli->enable_map);
if (errcode)
goto error;
phys_em.fe_bm = setup->fe_bm;
phys_em.shader_bm = setup->shader_bm;
phys_em.tiler_bm = setup->tiler_bm;
phys_em.mmu_l2_bm = setup->mmu_l2_bm;
kbase_hwcnt_gpu_enable_map_from_physical(&vcli->enable_map, &phys_em);
/* Use virtualizer's metadata to alloc tmp buffer which interacts with
* the HWC virtualizer.
*/
errcode = kbase_hwcnt_dump_buffer_alloc(vctx->metadata, &vcli->tmp_buf);
if (errcode)
goto error;
/* Enable all the available clk_enable_map. */
vcli->enable_map.clk_enable_map = (1ull << vctx->metadata->clk_cnt) - 1;
/* Use vinstr's narrowed metadata to alloc narrow dump buffers which
* interact with clients.
*/
errcode = kbase_hwcnt_dump_buffer_narrow_array_alloc(
vctx->metadata_user, setup->buffer_count, &vcli->dump_bufs);
if (errcode)
goto error;
errcode = -ENOMEM;
vcli->dump_bufs_meta = kmalloc_array(
setup->buffer_count, sizeof(*vcli->dump_bufs_meta), GFP_KERNEL);
if (!vcli->dump_bufs_meta)
goto error;
errcode = kbase_hwcnt_virtualizer_client_create(
vctx->hvirt, &vcli->enable_map, &vcli->hvcli);
if (errcode)
goto error;
init_waitqueue_head(&vcli->waitq);
*out_vcli = vcli;
return 0;
error:
kbasep_vinstr_client_destroy(vcli);
return errcode;
}
int kbase_vinstr_init(
struct kbase_hwcnt_virtualizer *hvirt,
struct kbase_vinstr_context **out_vctx)
{
int errcode;
struct kbase_vinstr_context *vctx;
const struct kbase_hwcnt_metadata *metadata;
if (!hvirt || !out_vctx)
return -EINVAL;
metadata = kbase_hwcnt_virtualizer_metadata(hvirt);
if (!metadata)
return -EINVAL;
vctx = kzalloc(sizeof(*vctx), GFP_KERNEL);
if (!vctx)
return -ENOMEM;
vctx->hvirt = hvirt;
vctx->metadata = metadata;
errcode = kbase_hwcnt_gpu_metadata_narrow_create(&vctx->metadata_user,
metadata);
if (errcode)
goto err_metadata_create;
mutex_init(&vctx->lock);
INIT_LIST_HEAD(&vctx->clients);
hrtimer_init(&vctx->dump_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
vctx->dump_timer.function = kbasep_vinstr_dump_timer;
INIT_WORK(&vctx->dump_work, kbasep_vinstr_dump_worker);
*out_vctx = vctx;
return 0;
err_metadata_create:
kfree(vctx);
return errcode;
}
void kbase_vinstr_term(struct kbase_vinstr_context *vctx)
{
if (!vctx)
return;
/* Non-zero client count implies client leak */
if (WARN_ON(vctx->client_count != 0)) {
struct kbase_vinstr_client *pos, *n;
list_for_each_entry_safe(pos, n, &vctx->clients, node) {
list_del(&pos->node);
vctx->client_count--;
kbasep_vinstr_client_destroy(pos);
}
}
cancel_work_sync(&vctx->dump_work);
kbase_hwcnt_gpu_metadata_narrow_destroy(vctx->metadata_user);
WARN_ON(vctx->client_count != 0);
kfree(vctx);
}
void kbase_vinstr_suspend(struct kbase_vinstr_context *vctx)
{
if (WARN_ON(!vctx))
return;
mutex_lock(&vctx->lock);
if (!WARN_ON(vctx->suspend_count == SIZE_MAX))
vctx->suspend_count++;
mutex_unlock(&vctx->lock);
/* Always sync cancel the timer and then the worker, regardless of the
* new suspend count.
*
* This ensures concurrent calls to kbase_vinstr_suspend() always block
* until vinstr is fully suspended.
*
* The timer is cancelled before the worker, as the timer
* unconditionally re-enqueues the worker, but the worker checks the
* suspend_count that we just incremented before rescheduling the timer.
*
* Therefore if we cancel the worker first, the timer might re-enqueue
* the worker before we cancel the timer, but the opposite is not
* possible.
*/
hrtimer_cancel(&vctx->dump_timer);
cancel_work_sync(&vctx->dump_work);
}
void kbase_vinstr_resume(struct kbase_vinstr_context *vctx)
{
if (WARN_ON(!vctx))
return;
mutex_lock(&vctx->lock);
if (!WARN_ON(vctx->suspend_count == 0)) {
vctx->suspend_count--;
/* Last resume, so re-enqueue the worker if we have any periodic
* clients.
*/
if (vctx->suspend_count == 0) {
struct kbase_vinstr_client *pos;
bool has_periodic_clients = false;
list_for_each_entry(pos, &vctx->clients, node) {
if (pos->dump_interval_ns != 0) {
has_periodic_clients = true;
break;
}
}
if (has_periodic_clients)
kbase_hwcnt_virtualizer_queue_work(
vctx->hvirt, &vctx->dump_work);
}
}
mutex_unlock(&vctx->lock);
}
int kbase_vinstr_hwcnt_reader_setup(
struct kbase_vinstr_context *vctx,
struct kbase_ioctl_hwcnt_reader_setup *setup)
{
int errcode;
int fd;
struct kbase_vinstr_client *vcli = NULL;
if (!vctx || !setup ||
(setup->buffer_count == 0) ||
(setup->buffer_count > MAX_BUFFER_COUNT) ||
!is_power_of_2(setup->buffer_count))
return -EINVAL;
errcode = kbasep_vinstr_client_create(vctx, setup, &vcli);
if (errcode)
goto error;
/* Add the new client. No need to reschedule worker, as not periodic */
mutex_lock(&vctx->lock);
vctx->client_count++;
list_add(&vcli->node, &vctx->clients);
mutex_unlock(&vctx->lock);
/* Expose to user-space only once the client is fully initialized */
errcode = anon_inode_getfd(
"[mali_vinstr_desc]",
&vinstr_client_fops,
vcli,
O_RDONLY | O_CLOEXEC);
if (errcode < 0)
goto client_installed_error;
fd = errcode;
return fd;
client_installed_error:
mutex_lock(&vctx->lock);
vctx->client_count--;
list_del(&vcli->node);
mutex_unlock(&vctx->lock);
error:
kbasep_vinstr_client_destroy(vcli);
return errcode;
}
/**
* kbasep_vinstr_hwcnt_reader_buffer_ready() - Check if client has ready
* buffers.
* @cli: Non-NULL pointer to vinstr client.
*
* Return: Non-zero if client has at least one dumping buffer filled that was
* not notified to user yet.
*/
static int kbasep_vinstr_hwcnt_reader_buffer_ready(
struct kbase_vinstr_client *cli)
{
WARN_ON(!cli);
return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx);
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_dump() - Dump ioctl command.
* @cli: Non-NULL pointer to vinstr client.
*
* Return: 0 on success, else error code.
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_dump(
struct kbase_vinstr_client *cli)
{
int errcode;
mutex_lock(&cli->vctx->lock);
errcode = kbasep_vinstr_client_dump(
cli, BASE_HWCNT_READER_EVENT_MANUAL);
mutex_unlock(&cli->vctx->lock);
return errcode;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_clear() - Clear ioctl command.
* @cli: Non-NULL pointer to vinstr client.
*
* Return: 0 on success, else error code.
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_clear(
struct kbase_vinstr_client *cli)
{
int errcode;
mutex_lock(&cli->vctx->lock);
errcode = kbasep_vinstr_client_clear(cli);
mutex_unlock(&cli->vctx->lock);
return errcode;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_get_buffer() - Get buffer ioctl command.
* @cli: Non-NULL pointer to vinstr client.
* @buffer: Non-NULL pointer to userspace buffer.
* @size: Size of buffer.
*
* Return: 0 on success, else error code.
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
struct kbase_vinstr_client *cli,
void __user *buffer,
size_t size)
{
unsigned int meta_idx = atomic_read(&cli->meta_idx);
unsigned int idx = meta_idx % cli->dump_bufs.buf_cnt;
struct kbase_hwcnt_reader_metadata *meta = &cli->dump_bufs_meta[idx];
const size_t meta_size = sizeof(struct kbase_hwcnt_reader_metadata);
const size_t min_size = min(size, meta_size);
/* Metadata sanity check. */
WARN_ON(idx != meta->buffer_idx);
/* Check if there is any buffer available. */
if (unlikely(atomic_read(&cli->write_idx) == meta_idx))
return -EAGAIN;
/* Check if previously taken buffer was put back. */
if (unlikely(atomic_read(&cli->read_idx) != meta_idx))
return -EBUSY;
/* Clear user buffer to zero. */
if (unlikely(meta_size < size && clear_user(buffer, size)))
return -EFAULT;
/* Copy next available buffer's metadata to user. */
if (unlikely(copy_to_user(buffer, meta, min_size)))
return -EFAULT;
/* Compare exchange meta idx to protect against concurrent getters */
if (meta_idx != atomic_cmpxchg(&cli->meta_idx, meta_idx, meta_idx + 1))
return -EBUSY;
return 0;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_put_buffer() - Put buffer ioctl command.
* @cli: Non-NULL pointer to vinstr client.
* @buffer: Non-NULL pointer to userspace buffer.
* @size: Size of buffer.
*
* Return: 0 on success, else error code.
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
struct kbase_vinstr_client *cli,
void __user *buffer,
size_t size)
{
unsigned int read_idx = atomic_read(&cli->read_idx);
unsigned int idx = read_idx % cli->dump_bufs.buf_cnt;
struct kbase_hwcnt_reader_metadata *meta;
const size_t meta_size = sizeof(struct kbase_hwcnt_reader_metadata);
const size_t max_size = max(size, meta_size);
int ret = 0;
u8 stack_kbuf[64];
u8 *kbuf = NULL;
size_t i;
/* Check if any buffer was taken. */
if (unlikely(atomic_read(&cli->meta_idx) == read_idx))
return -EPERM;
if (likely(max_size <= sizeof(stack_kbuf))) {
/* Use stack buffer when the size is small enough. */
if (unlikely(meta_size > size))
memset(stack_kbuf, 0, sizeof(stack_kbuf));
kbuf = stack_kbuf;
} else {
kbuf = kzalloc(max_size, GFP_KERNEL);
if (unlikely(!kbuf))
return -ENOMEM;
}
/*
* Copy user buffer to zero cleared kernel buffer which has enough
* space for both user buffer and kernel metadata.
*/
if (unlikely(copy_from_user(kbuf, buffer, size))) {
ret = -EFAULT;
goto out;
}
/*
* Make sure any "extra" data passed from userspace is zero.
* It's meaningful only in case meta_size < size.
*/
for (i = meta_size; i < size; i++) {
/* Check if user data beyond meta size is zero. */
if (unlikely(kbuf[i] != 0)) {
ret = -EINVAL;
goto out;
}
}
/* Check if correct buffer is put back. */
meta = (struct kbase_hwcnt_reader_metadata *)kbuf;
if (unlikely(idx != meta->buffer_idx)) {
ret = -EINVAL;
goto out;
}
/* Compare exchange read idx to protect against concurrent putters */
if (read_idx !=
atomic_cmpxchg(&cli->read_idx, read_idx, read_idx + 1)) {
ret = -EPERM;
goto out;
}
out:
if (unlikely(kbuf != stack_kbuf))
kfree(kbuf);
return ret;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_set_interval() - Set interval ioctl command.
* @cli: Non-NULL pointer to vinstr client.
* @interval: Periodic dumping interval (disable periodic dumping if 0).
*
* Return: 0 always.
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
struct kbase_vinstr_client *cli,
u32 interval)
{
mutex_lock(&cli->vctx->lock);
if ((interval != 0) && (interval < DUMP_INTERVAL_MIN_NS))
interval = DUMP_INTERVAL_MIN_NS;
/* Update the interval, and put in a dummy next dump time */
cli->dump_interval_ns = interval;
cli->next_dump_time_ns = 0;
/*
* If it's a periodic client, kick off the worker early to do a proper
* timer reschedule. Return value is ignored, as we don't care if the
* worker is already queued.
*/
if ((interval != 0) && (cli->vctx->suspend_count == 0))
kbase_hwcnt_virtualizer_queue_work(cli->vctx->hvirt,
&cli->vctx->dump_work);
mutex_unlock(&cli->vctx->lock);
return 0;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_enable_event() - Enable event ioctl command.
* @cli: Non-NULL pointer to vinstr client.
* @event_id: ID of event to enable.
*
* Return: 0 always.
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
struct kbase_vinstr_client *cli,
enum base_hwcnt_reader_event event_id)
{
/* No-op, as events aren't supported */
return 0;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_disable_event() - Disable event ioctl
* command.
* @cli: Non-NULL pointer to vinstr client.
* @event_id: ID of event to disable.
*
* Return: 0 always.
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
struct kbase_vinstr_client *cli,
enum base_hwcnt_reader_event event_id)
{
/* No-op, as events aren't supported */
return 0;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_get_hwver() - Get HW version ioctl command.
* @cli: Non-NULL pointer to vinstr client.
* @hwver: Non-NULL pointer to user buffer where HW version will be stored.
*
* Return: 0 on success, else error code.
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
struct kbase_vinstr_client *cli,
u32 __user *hwver)
{
u32 ver = 5;
const enum kbase_hwcnt_gpu_group_type type =
kbase_hwcnt_metadata_group_type(cli->vctx->metadata, 0);
if (WARN_ON(type != KBASE_HWCNT_GPU_GROUP_TYPE_V5))
return -EINVAL;
return put_user(ver, hwver);
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl_get_api_version() - get API version ioctl
* command.
* @cli: The non-NULL pointer to the client
* @arg: Command's argument.
* @size: Size of arg.
*
* Return: 0 on success, else error code.
*/
static long kbasep_vinstr_hwcnt_reader_ioctl_get_api_version(
struct kbase_vinstr_client *cli, unsigned long arg, size_t size)
{
long ret = -EINVAL;
if (size == sizeof(u32)) {
ret = put_user(HWCNT_READER_API, (u32 __user *)arg);
} else if (size == sizeof(struct kbase_hwcnt_reader_api_version)) {
u8 clk_cnt = cli->vctx->metadata->clk_cnt;
unsigned long bytes = 0;
struct kbase_hwcnt_reader_api_version api_version = {
.version = HWCNT_READER_API,
.features = KBASE_HWCNT_READER_API_VERSION_NO_FEATURE,
};
if (clk_cnt > 0)
api_version.features |=
KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_TOP;
if (clk_cnt > 1)
api_version.features |=
KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_SHADER_CORES;
bytes = copy_to_user(
(void __user *)arg, &api_version, sizeof(api_version));
/* copy_to_user returns zero in case of success.
* If it fails, it returns the number of bytes that could NOT be copied
*/
if (bytes == 0)
ret = 0;
else
ret = -EFAULT;
}
return ret;
}
/**
* kbasep_vinstr_hwcnt_reader_ioctl() - hwcnt reader's ioctl.
* @filp: Non-NULL pointer to file structure.
* @cmd: User command.
* @arg: Command's argument.
*
* Return: 0 on success, else error code.
*/
static long kbasep_vinstr_hwcnt_reader_ioctl(
struct file *filp,
unsigned int cmd,
unsigned long arg)
{
long rcode;
struct kbase_vinstr_client *cli;
if (!filp || (_IOC_TYPE(cmd) != KBASE_HWCNT_READER))
return -EINVAL;
cli = filp->private_data;
if (!cli)
return -EINVAL;
switch (_IOC_NR(cmd)) {
case _IOC_NR(KBASE_HWCNT_READER_GET_API_VERSION):
rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_api_version(
cli, arg, _IOC_SIZE(cmd));
break;
case _IOC_NR(KBASE_HWCNT_READER_GET_HWVER):
rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
cli, (u32 __user *)arg);
break;
case _IOC_NR(KBASE_HWCNT_READER_GET_BUFFER_SIZE):
rcode = put_user((u32)cli->vctx->metadata_user->dump_buf_bytes,
(u32 __user *)arg);
break;
case _IOC_NR(KBASE_HWCNT_READER_DUMP):
rcode = kbasep_vinstr_hwcnt_reader_ioctl_dump(cli);
break;
case _IOC_NR(KBASE_HWCNT_READER_CLEAR):
rcode = kbasep_vinstr_hwcnt_reader_ioctl_clear(cli);
break;
case _IOC_NR(KBASE_HWCNT_READER_GET_BUFFER):
rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
cli, (void __user *)arg, _IOC_SIZE(cmd));
break;
case _IOC_NR(KBASE_HWCNT_READER_PUT_BUFFER):
rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
cli, (void __user *)arg, _IOC_SIZE(cmd));
break;
case _IOC_NR(KBASE_HWCNT_READER_SET_INTERVAL):
rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
cli, (u32)arg);
break;
case _IOC_NR(KBASE_HWCNT_READER_ENABLE_EVENT):
rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
cli, (enum base_hwcnt_reader_event)arg);
break;
case _IOC_NR(KBASE_HWCNT_READER_DISABLE_EVENT):
rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
cli, (enum base_hwcnt_reader_event)arg);
break;
default:
pr_warn("Unknown HWCNT ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd));
rcode = -EINVAL;
break;
}
return rcode;
}
/**
* kbasep_vinstr_hwcnt_reader_poll() - hwcnt reader's poll.
* @filp: Non-NULL pointer to file structure.
* @wait: Non-NULL pointer to poll table.
*
* Return: POLLIN if data can be read without blocking, 0 if data can not be
* read without blocking, else error code.
*/
static __poll_t kbasep_vinstr_hwcnt_reader_poll(struct file *filp, poll_table *wait)
{
struct kbase_vinstr_client *cli;
if (!filp || !wait)
return (__poll_t)-EINVAL;
cli = filp->private_data;
if (!cli)
return (__poll_t)-EINVAL;
poll_wait(filp, &cli->waitq, wait);
if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli))
return POLLIN;
return 0;
}
/**
* kbasep_vinstr_hwcnt_reader_mmap() - hwcnt reader's mmap.
* @filp: Non-NULL pointer to file structure.
* @vma: Non-NULL pointer to vma structure.
*
* Return: 0 on success, else error code.
*/
static int kbasep_vinstr_hwcnt_reader_mmap(
struct file *filp,
struct vm_area_struct *vma)
{
struct kbase_vinstr_client *cli;
unsigned long vm_size, size, addr, pfn, offset;
if (!filp || !vma)
return -EINVAL;
cli = filp->private_data;
if (!cli)
return -EINVAL;
vm_size = vma->vm_end - vma->vm_start;
/* The mapping is allowed to span the entirety of the page allocation,
* not just the chunk where the dump buffers are allocated.
* This accommodates the corner case where the combined size of the
* dump buffers is smaller than a single page.
* This does not pose a security risk as the pages are zeroed on
* allocation, and anything out of bounds of the dump buffers is never
* written to.
*/
size = (1ull << cli->dump_bufs.page_order) * PAGE_SIZE;
if (vma->vm_pgoff > (size >> PAGE_SHIFT))
return -EINVAL;
offset = vma->vm_pgoff << PAGE_SHIFT;
if (vm_size > size - offset)
return -EINVAL;
addr = __pa(cli->dump_bufs.page_addr + offset);
pfn = addr >> PAGE_SHIFT;
return remap_pfn_range(
vma, vma->vm_start, pfn, vm_size, vma->vm_page_prot);
}
/**
* kbasep_vinstr_hwcnt_reader_release() - hwcnt reader's release.
* @inode: Non-NULL pointer to inode structure.
* @filp: Non-NULL pointer to file structure.
*
* Return: 0 always.
*/
static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode,
struct file *filp)
{
struct kbase_vinstr_client *vcli = filp->private_data;
mutex_lock(&vcli->vctx->lock);
vcli->vctx->client_count--;
list_del(&vcli->node);
mutex_unlock(&vcli->vctx->lock);
kbasep_vinstr_client_destroy(vcli);
return 0;
}
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags