Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2)  *   This file is provided under a dual BSD/GPLv2 license.  When using or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *   redistributing this file, you may do so under either license.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *   GPL LICENSE SUMMARY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *   Copyright (C) 2017 T-Platforms. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  *   This program is free software; you can redistribute it and/or modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  *   it under the terms of version 2 of the GNU General Public License as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  *   published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  *   This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  *   WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)  *   General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  *   BSD LICENSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22)  *   Copyright (C) 2017 T-Platforms. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24)  *   Redistribution and use in source and binary forms, with or without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25)  *   modification, are permitted provided that the following conditions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26)  *   are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  *     * Redistributions of source code must retain the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  *       notice, this list of conditions and the following disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30)  *     * Redistributions in binary form must reproduce the above copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31)  *       notice, this list of conditions and the following disclaimer in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32)  *       the documentation and/or other materials provided with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33)  *       distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34)  *     * Neither the name of Intel Corporation nor the names of its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35)  *       contributors may be used to endorse or promote products derived
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36)  *       from this software without specific prior written permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38)  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39)  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40)  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41)  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42)  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43)  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44)  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45)  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46)  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47)  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48)  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50)  * PCIe NTB Pingpong Linux driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54)  * How to use this tool, by example.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56)  * Assuming $DBG_DIR is something like:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57)  * '/sys/kernel/debug/ntb_perf/0000:00:03.0'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58)  * Suppose aside from local device there is at least one remote device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59)  * connected to NTB with index 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60)  *-----------------------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61)  * Eg: install driver with specified delay between doorbell event and response
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63)  * root@self# insmod ntb_pingpong.ko delay_ms=1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64)  *-----------------------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65)  * Eg: get number of ping-pong cycles performed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67)  * root@self# cat $DBG_DIR/count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) #include <linux/hrtimer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) #include <linux/ntb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) #define DRIVER_NAME		"ntb_pingpong"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) #define DRIVER_VERSION		"2.0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) MODULE_LICENSE("Dual BSD/GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) MODULE_VERSION(DRIVER_VERSION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) MODULE_AUTHOR("Allen Hubbe <Allen.Hubbe@emc.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) MODULE_DESCRIPTION("PCIe NTB Simple Pingpong Client");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) static unsigned int unsafe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) module_param(unsafe, uint, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) MODULE_PARM_DESC(unsafe, "Run even though ntb operations may be unsafe");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) static unsigned int delay_ms = 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) module_param(delay_ms, uint, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) MODULE_PARM_DESC(delay_ms, "Milliseconds to delay the response to peer");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) struct pp_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	struct ntb_dev *ntb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	struct hrtimer timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	u64 in_db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	u64 out_db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	int out_pidx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	u64 nmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	u64 pmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	atomic_t count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	struct dentry *dbgfs_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define to_pp_timer(__timer) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	container_of(__timer, struct pp_ctx, timer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) static struct dentry *pp_dbgfs_topdir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) static int pp_find_next_peer(struct pp_ctx *pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	u64 link, out_db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	int pidx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	link = ntb_link_is_up(pp->ntb, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	/* Find next available peer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	if (link & pp->nmask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 		pidx = __ffs64(link & pp->nmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	else if (link & pp->pmask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 		pidx = __ffs64(link & pp->pmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	out_db = BIT_ULL(ntb_peer_port_number(pp->ntb, pidx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	spin_lock(&pp->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	pp->out_pidx = pidx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	pp->out_db = out_db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	spin_unlock(&pp->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) static void pp_setup(struct pp_ctx *pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	ntb_db_set_mask(pp->ntb, pp->in_db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	hrtimer_cancel(&pp->timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	ret = pp_find_next_peer(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	if (ret == -ENODEV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 		dev_dbg(&pp->ntb->dev, "Got no peers, so cancel\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	dev_dbg(&pp->ntb->dev, "Ping-pong started with port %d, db %#llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 		ntb_peer_port_number(pp->ntb, pp->out_pidx), pp->out_db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	hrtimer_start(&pp->timer, ms_to_ktime(delay_ms), HRTIMER_MODE_REL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) static void pp_clear(struct pp_ctx *pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	hrtimer_cancel(&pp->timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	ntb_db_set_mask(pp->ntb, pp->in_db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	dev_dbg(&pp->ntb->dev, "Ping-pong cancelled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) static void pp_ping(struct pp_ctx *pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	u32 count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	count = atomic_read(&pp->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	spin_lock(&pp->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	ntb_peer_spad_write(pp->ntb, pp->out_pidx, 0, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	ntb_peer_msg_write(pp->ntb, pp->out_pidx, 0, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	dev_dbg(&pp->ntb->dev, "Ping port %d spad %#x, msg %#x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 		ntb_peer_port_number(pp->ntb, pp->out_pidx), count, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	ntb_peer_db_set(pp->ntb, pp->out_db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	ntb_db_clear_mask(pp->ntb, pp->in_db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	spin_unlock(&pp->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static void pp_pong(struct pp_ctx *pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	u32 msg_data = -1, spad_data = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	int pidx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	/* Read pong data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	spad_data = ntb_spad_read(pp->ntb, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	msg_data = ntb_msg_read(pp->ntb, &pidx, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	ntb_msg_clear_sts(pp->ntb, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	 * Scratchpad and message data may differ, since message register can't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	 * be rewritten unless status is cleared. Additionally either of them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	 * might be unsupported
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	dev_dbg(&pp->ntb->dev, "Pong spad %#x, msg %#x (port %d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 		spad_data, msg_data, ntb_peer_port_number(pp->ntb, pidx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	atomic_inc(&pp->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	ntb_db_set_mask(pp->ntb, pp->in_db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	ntb_db_clear(pp->ntb, pp->in_db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	hrtimer_start(&pp->timer, ms_to_ktime(delay_ms), HRTIMER_MODE_REL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) static enum hrtimer_restart pp_timer_func(struct hrtimer *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	struct pp_ctx *pp = to_pp_timer(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	pp_ping(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	return HRTIMER_NORESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) static void pp_link_event(void *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	struct pp_ctx *pp = ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	pp_setup(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static void pp_db_event(void *ctx, int vec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	struct pp_ctx *pp = ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	pp_pong(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) static const struct ntb_ctx_ops pp_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	.link_event = pp_link_event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	.db_event = pp_db_event
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) static int pp_check_ntb(struct ntb_dev *ntb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	u64 pmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	if (ntb_db_is_unsafe(ntb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 		dev_dbg(&ntb->dev, "Doorbell is unsafe\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 		if (!unsafe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 			return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	if (ntb_spad_is_unsafe(ntb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 		dev_dbg(&ntb->dev, "Scratchpad is unsafe\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 		if (!unsafe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 			return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	pmask = GENMASK_ULL(ntb_peer_port_count(ntb), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	if ((ntb_db_valid_mask(ntb) & pmask) != pmask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 		dev_err(&ntb->dev, "Unsupported DB configuration\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	if (ntb_spad_count(ntb) < 1 && ntb_msg_count(ntb) < 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 		dev_err(&ntb->dev, "Scratchpads and messages unsupported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	} else if (ntb_spad_count(ntb) < 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 		dev_dbg(&ntb->dev, "Scratchpads unsupported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	} else if (ntb_msg_count(ntb) < 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 		dev_dbg(&ntb->dev, "Messages unsupported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static struct pp_ctx *pp_create_data(struct ntb_dev *ntb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	struct pp_ctx *pp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	pp = devm_kzalloc(&ntb->dev, sizeof(*pp), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	if (!pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 		return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	pp->ntb = ntb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	atomic_set(&pp->count, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	spin_lock_init(&pp->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	hrtimer_init(&pp->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	pp->timer.function = pp_timer_func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	return pp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) static void pp_init_flds(struct pp_ctx *pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	int pidx, lport, pcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	/* Find global port index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	lport = ntb_port_number(pp->ntb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	pcnt = ntb_peer_port_count(pp->ntb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	for (pidx = 0; pidx < pcnt; pidx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 		if (lport < ntb_peer_port_number(pp->ntb, pidx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	pp->in_db = BIT_ULL(lport);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	pp->pmask = GENMASK_ULL(pidx, 0) >> 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	pp->nmask = GENMASK_ULL(pcnt - 1, pidx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	dev_dbg(&pp->ntb->dev, "Inbound db %#llx, prev %#llx, next %#llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 		pp->in_db, pp->pmask, pp->nmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) static int pp_mask_events(struct pp_ctx *pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	u64 db_mask, msg_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	db_mask = ntb_db_valid_mask(pp->ntb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	ret = ntb_db_set_mask(pp->ntb, db_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 		return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	/* Skip message events masking if unsupported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 	if (ntb_msg_count(pp->ntb) < 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	msg_mask = ntb_msg_outbits(pp->ntb) | ntb_msg_inbits(pp->ntb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	return ntb_msg_set_mask(pp->ntb, msg_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) static int pp_setup_ctx(struct pp_ctx *pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 	int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 	ret = ntb_set_ctx(pp->ntb, pp, &pp_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 	if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 		return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 	ntb_link_enable(pp->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 	/* Might be not necessary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 	ntb_link_event(pp->ntb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) static void pp_clear_ctx(struct pp_ctx *pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 	ntb_link_disable(pp->ntb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 	ntb_clear_ctx(pp->ntb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static void pp_setup_dbgfs(struct pp_ctx *pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 	struct pci_dev *pdev = pp->ntb->pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 	pp->dbgfs_dir = debugfs_create_dir(pci_name(pdev), pp_dbgfs_topdir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	debugfs_create_atomic_t("count", 0600, pp->dbgfs_dir, &pp->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) static void pp_clear_dbgfs(struct pp_ctx *pp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 	debugfs_remove_recursive(pp->dbgfs_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) static int pp_probe(struct ntb_client *client, struct ntb_dev *ntb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	struct pp_ctx *pp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 	int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	ret = pp_check_ntb(ntb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 		return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 	pp = pp_create_data(ntb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 	if (IS_ERR(pp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 		return PTR_ERR(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 	pp_init_flds(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 	ret = pp_mask_events(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 	if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 		return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 	ret = pp_setup_ctx(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 	if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 		return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 	pp_setup_dbgfs(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) static void pp_remove(struct ntb_client *client, struct ntb_dev *ntb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 	struct pp_ctx *pp = ntb->ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) 	pp_clear_dbgfs(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 	pp_clear_ctx(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) 	pp_clear(pp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static struct ntb_client pp_client = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 	.ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 		.probe = pp_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 		.remove = pp_remove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) static int __init pp_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) 	int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) 	if (debugfs_initialized())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 		pp_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) 	ret = ntb_register_client(&pp_client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) 	if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) 		debugfs_remove_recursive(pp_dbgfs_topdir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) module_init(pp_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) static void __exit pp_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) 	ntb_unregister_client(&pp_client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) 	debugfs_remove_recursive(pp_dbgfs_topdir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) module_exit(pp_exit);