^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* ldc.c: Logical Domain Channel link-layer protocol driver.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/scatterlist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/bitmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/iommu-common.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/hypervisor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/ldc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <asm/mdesc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define DRV_MODULE_NAME "ldc"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define PFX DRV_MODULE_NAME ": "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define DRV_MODULE_VERSION "1.1"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define DRV_MODULE_RELDATE "July 22, 2008"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define COOKIE_PGSZ_CODE 0xf000000000000000ULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define COOKIE_PGSZ_CODE_SHIFT 60ULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static char version[] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /* Packet header layout for unreliable and reliable mode frames.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * When in RAW mode, packets are simply straight 64-byte payloads
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * with no headers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct ldc_packet {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) u8 type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define LDC_CTRL 0x01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define LDC_DATA 0x02
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define LDC_ERR 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) u8 stype;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define LDC_INFO 0x01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define LDC_ACK 0x02
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define LDC_NACK 0x04
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) u8 ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define LDC_VERS 0x01 /* Link Version */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define LDC_RTS 0x02 /* Request To Send */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define LDC_RTR 0x03 /* Ready To Receive */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define LDC_RDX 0x04 /* Ready for Data eXchange */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define LDC_CTRL_MSK 0x0f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) u8 env;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define LDC_LEN 0x3f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define LDC_FRAG_MASK 0xc0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #define LDC_START 0x40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define LDC_STOP 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) u32 seqid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) u8 u_data[LDC_PACKET_SIZE - 8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) u32 pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) u32 ackid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) u8 r_data[LDC_PACKET_SIZE - 8 - 8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) } r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) } u;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct ldc_version {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) u16 major;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) u16 minor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /* Ordered from largest major to lowest. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static struct ldc_version ver_arr[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) { .major = 1, .minor = 0 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) struct ldc_channel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) struct ldc_mode_ops {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) int (*write)(struct ldc_channel *, const void *, unsigned int);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) int (*read)(struct ldc_channel *, void *, unsigned int);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static const struct ldc_mode_ops raw_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static const struct ldc_mode_ops nonraw_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) static const struct ldc_mode_ops stream_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) int ldom_domaining_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct ldc_iommu {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) /* Protects ldc_unmap. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) struct ldc_mtable_entry *page_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) struct iommu_map_table iommu_map_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) struct ldc_channel {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /* Protects all operations that depend upon channel state. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) unsigned long id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) u8 *mssbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) u32 mssbuf_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) u32 mssbuf_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) struct ldc_packet *tx_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) unsigned long tx_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) unsigned long tx_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) unsigned long tx_num_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) unsigned long tx_ra;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) unsigned long tx_acked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) struct ldc_packet *rx_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) unsigned long rx_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) unsigned long rx_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) unsigned long rx_num_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) unsigned long rx_ra;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) u32 rcv_nxt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) u32 snd_nxt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) unsigned long chan_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct ldc_channel_config cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) void *event_arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) const struct ldc_mode_ops *mops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct ldc_iommu iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct ldc_version ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) u8 hs_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #define LDC_HS_CLOSED 0x00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) #define LDC_HS_OPEN 0x01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #define LDC_HS_GOTVERS 0x02
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #define LDC_HS_SENTRTR 0x03
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) #define LDC_HS_GOTRTR 0x04
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #define LDC_HS_COMPLETE 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) u8 flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #define LDC_FLAG_ALLOCED_QUEUES 0x01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #define LDC_FLAG_REGISTERED_QUEUES 0x02
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) #define LDC_FLAG_REGISTERED_IRQS 0x04
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #define LDC_FLAG_RESET 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) u8 mss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) u8 state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) #define LDC_IRQ_NAME_MAX 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) char rx_irq_name[LDC_IRQ_NAME_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) char tx_irq_name[LDC_IRQ_NAME_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) struct hlist_head mh_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) struct hlist_node list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) #define ldcdbg(TYPE, f, a...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) #define LDC_ABORT(lp) ldc_abort((lp), __func__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static const char *state_to_str(u8 state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) switch (state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) case LDC_STATE_INVALID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) return "INVALID";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) case LDC_STATE_INIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return "INIT";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) case LDC_STATE_BOUND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) return "BOUND";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) case LDC_STATE_READY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) return "READY";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) case LDC_STATE_CONNECTED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) return "CONNECTED";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) return "<UNKNOWN>";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static unsigned long __advance(unsigned long off, unsigned long num_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) off += LDC_PACKET_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (off == (num_entries * LDC_PACKET_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) return off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) return __advance(off, lp->rx_num_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) return __advance(off, lp->tx_num_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) unsigned long *new_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) unsigned long t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) t = tx_advance(lp, lp->tx_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) if (t == lp->tx_head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) *new_tail = t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) p = lp->tx_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return p + (lp->tx_tail / LDC_PACKET_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /* When we are in reliable or stream mode, have to track the next packet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * we haven't gotten an ACK for in the TX queue using tx_acked. We have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * to be careful not to stomp over the queue past that point. During
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * the handshake, we don't have TX data packets pending in the queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * and that's why handshake_get_tx_packet() need not be mindful of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * lp->tx_acked.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) static unsigned long head_for_data(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (lp->cfg.mode == LDC_MODE_STREAM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) return lp->tx_acked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) return lp->tx_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) static int tx_has_space_for(struct ldc_channel *lp, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) unsigned long limit, tail, new_tail, diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) unsigned int mss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) limit = head_for_data(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) tail = lp->tx_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) new_tail = tx_advance(lp, tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (new_tail == limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (limit > new_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) diff = limit - new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) diff = (limit +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) diff /= LDC_PACKET_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) mss = lp->mss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) if (diff * mss < size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) unsigned long *new_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) unsigned long h, t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) h = head_for_data(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) t = tx_advance(lp, lp->tx_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) if (t == h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) *new_tail = t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) p = lp->tx_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) return p + (lp->tx_tail / LDC_PACKET_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) static int set_tx_tail(struct ldc_channel *lp, unsigned long tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) unsigned long orig_tail = lp->tx_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) int limit = 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) lp->tx_tail = tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) while (limit-- > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) unsigned long err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) err = sun4v_ldc_tx_set_qtail(lp->id, tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (err != HV_EWOULDBLOCK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) lp->tx_tail = orig_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) udelay(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) lp->tx_tail = orig_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) /* This just updates the head value in the hypervisor using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * a polling loop with a timeout. The caller takes care of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * upating software state representing the head change, if any.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) static int __set_rx_head(struct ldc_channel *lp, unsigned long head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) int limit = 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) while (limit-- > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) unsigned long err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) err = sun4v_ldc_rx_set_qhead(lp->id, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) if (err != HV_EWOULDBLOCK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) udelay(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) static int send_tx_packet(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) struct ldc_packet *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) unsigned long new_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return set_tx_tail(lp, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) u8 stype, u8 ctrl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) void *data, int dlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) unsigned long *new_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) memset(p, 0, sizeof(*p));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) p->type = LDC_CTRL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) p->stype = stype;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) p->ctrl = ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) memcpy(p->u.u_data, data, dlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) return p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) static int start_handshake(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) struct ldc_version *ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) unsigned long new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) ver = &ver_arr[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) ver->major, ver->minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) ver, sizeof(*ver), &new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) int err = send_tx_packet(lp, p, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) lp->flags &= ~LDC_FLAG_RESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) static int send_version_nack(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) u16 major, u16 minor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) struct ldc_version ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) unsigned long new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) ver.major = major;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) ver.minor = minor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) &ver, sizeof(ver), &new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) ver.major, ver.minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) return send_tx_packet(lp, p, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) static int send_version_ack(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) struct ldc_version *vp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) unsigned long new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) vp, sizeof(*vp), &new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) vp->major, vp->minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) return send_tx_packet(lp, p, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) static int send_rts(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) unsigned long new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) &new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) p->env = lp->cfg.mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) p->seqid = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) lp->rcv_nxt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) p->env, p->seqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) return send_tx_packet(lp, p, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) static int send_rtr(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) unsigned long new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) &new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) p->env = lp->cfg.mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) p->seqid = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) p->env, p->seqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return send_tx_packet(lp, p, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) static int send_rdx(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) unsigned long new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) &new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) p->env = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) p->seqid = ++lp->snd_nxt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) p->u.r.ackid = lp->rcv_nxt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) p->env, p->seqid, p->u.r.ackid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return send_tx_packet(lp, p, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) unsigned long new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) p = data_get_tx_packet(lp, &new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) memset(p, 0, sizeof(*p));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) p->type = data_pkt->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) p->stype = LDC_NACK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) p->seqid = lp->snd_nxt + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) p->u.r.ackid = lp->rcv_nxt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) p->type, p->ctrl, p->seqid, p->u.r.ackid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) err = send_tx_packet(lp, p, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) lp->snd_nxt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) static int ldc_abort(struct ldc_channel *lp, const char *msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) unsigned long hv_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) ldcdbg(STATE, "ABORT[%s]\n", msg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) ldc_print(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) /* We report but do not act upon the hypervisor errors because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) * there really isn't much we can do if they fail at this point.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) printk(KERN_ERR PFX "ldc_abort: "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) lp->id, lp->tx_ra, lp->tx_num_entries, hv_err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) hv_err = sun4v_ldc_tx_get_state(lp->id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) &lp->tx_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) &lp->tx_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) &lp->chan_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) printk(KERN_ERR PFX "ldc_abort: "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) lp->id, hv_err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) printk(KERN_ERR PFX "ldc_abort: "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) lp->id, lp->rx_ra, lp->rx_num_entries, hv_err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) /* Refetch the RX queue state as well, because we could be invoked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) * here in the queue processing context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) hv_err = sun4v_ldc_rx_get_state(lp->id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) &lp->rx_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) &lp->rx_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) &lp->chan_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) printk(KERN_ERR PFX "ldc_abort: "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) lp->id, hv_err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) return -ECONNRESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) static struct ldc_version *find_by_major(u16 major)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) struct ldc_version *ret = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) for (i = 0; i < ARRAY_SIZE(ver_arr); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct ldc_version *v = &ver_arr[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if (v->major <= major) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) ret = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) struct ldc_version *vap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) vp->major, vp->minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) if (lp->hs_state == LDC_HS_GOTVERS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) lp->hs_state = LDC_HS_OPEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) memset(&lp->ver, 0, sizeof(lp->ver));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) vap = find_by_major(vp->major);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (!vap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) err = send_version_nack(lp, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) } else if (vap->major != vp->major) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) err = send_version_nack(lp, vap->major, vap->minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) struct ldc_version ver = *vp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) if (ver.minor > vap->minor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) ver.minor = vap->minor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) err = send_version_ack(lp, &ver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) lp->ver = ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) lp->hs_state = LDC_HS_GOTVERS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) vp->major, vp->minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) if (lp->hs_state == LDC_HS_GOTVERS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) if (lp->ver.major != vp->major ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) lp->ver.minor != vp->minor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) lp->ver = *vp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) lp->hs_state = LDC_HS_GOTVERS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) if (send_rts(lp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) struct ldc_version *vap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) unsigned long new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) if (vp->major == 0 && vp->minor == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) vap = find_by_major(vp->major);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) if (!vap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) vap, sizeof(*vap),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) &new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) return send_tx_packet(lp, p, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) static int process_version(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) struct ldc_packet *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) struct ldc_version *vp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) vp = (struct ldc_version *) p->u.u_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) switch (p->stype) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) case LDC_INFO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) return process_ver_info(lp, vp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) case LDC_ACK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) return process_ver_ack(lp, vp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) case LDC_NACK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) return process_ver_nack(lp, vp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) static int process_rts(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) struct ldc_packet *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) p->stype, p->seqid, p->env);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) if (p->stype != LDC_INFO ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) lp->hs_state != LDC_HS_GOTVERS ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) p->env != lp->cfg.mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) lp->snd_nxt = p->seqid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) lp->rcv_nxt = p->seqid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) lp->hs_state = LDC_HS_SENTRTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) if (send_rtr(lp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) static int process_rtr(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) struct ldc_packet *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) p->stype, p->seqid, p->env);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) if (p->stype != LDC_INFO ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) p->env != lp->cfg.mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) lp->snd_nxt = p->seqid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) lp->hs_state = LDC_HS_COMPLETE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) ldc_set_state(lp, LDC_STATE_CONNECTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) send_rdx(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) return LDC_EVENT_UP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) static int rx_seq_ok(struct ldc_channel *lp, u32 seqid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) return lp->rcv_nxt + 1 == seqid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) static int process_rdx(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) struct ldc_packet *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) p->stype, p->seqid, p->env, p->u.r.ackid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) if (p->stype != LDC_INFO ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) !(rx_seq_ok(lp, p->seqid)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) lp->rcv_nxt = p->seqid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) lp->hs_state = LDC_HS_COMPLETE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) ldc_set_state(lp, LDC_STATE_CONNECTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) return LDC_EVENT_UP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) static int process_control_frame(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) struct ldc_packet *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) switch (p->ctrl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) case LDC_VERS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) return process_version(lp, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) case LDC_RTS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) return process_rts(lp, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) case LDC_RTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) return process_rtr(lp, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) case LDC_RDX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) return process_rdx(lp, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) static int process_error_frame(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) struct ldc_packet *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) static int process_data_ack(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) struct ldc_packet *ack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) unsigned long head = lp->tx_acked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) u32 ackid = ack->u.r.ackid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) head = tx_advance(lp, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (p->seqid == ackid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) lp->tx_acked = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) if (head == lp->tx_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) static void send_events(struct ldc_channel *lp, unsigned int event_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) if (event_mask & LDC_EVENT_RESET)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) lp->cfg.event(lp->event_arg, LDC_EVENT_RESET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (event_mask & LDC_EVENT_UP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) lp->cfg.event(lp->event_arg, LDC_EVENT_UP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) if (event_mask & LDC_EVENT_DATA_READY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) static irqreturn_t ldc_rx(int irq, void *dev_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) struct ldc_channel *lp = dev_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) unsigned long orig_state, flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) unsigned int event_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) spin_lock_irqsave(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) orig_state = lp->chan_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) /* We should probably check for hypervisor errors here and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * reset the LDC channel if we get one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) sun4v_ldc_rx_get_state(lp->id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) &lp->rx_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) &lp->rx_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) &lp->chan_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) orig_state, lp->chan_state, lp->rx_head, lp->rx_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) event_mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) if (lp->cfg.mode == LDC_MODE_RAW &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) lp->chan_state == LDC_CHANNEL_UP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) lp->hs_state = LDC_HS_COMPLETE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) ldc_set_state(lp, LDC_STATE_CONNECTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) * Generate an LDC_EVENT_UP event if the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) * was not already up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) if (orig_state != LDC_CHANNEL_UP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) event_mask |= LDC_EVENT_UP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) orig_state = lp->chan_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) /* If we are in reset state, flush the RX queue and ignore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) * everything.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) if (lp->flags & LDC_FLAG_RESET) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) (void) ldc_rx_reset(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) /* Once we finish the handshake, we let the ldc_read()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * paths do all of the control frame and state management.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * Just trigger the callback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) if (lp->hs_state == LDC_HS_COMPLETE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) handshake_complete:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) if (lp->chan_state != orig_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) unsigned int event = LDC_EVENT_RESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) if (lp->chan_state == LDC_CHANNEL_UP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) event = LDC_EVENT_UP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) event_mask |= event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) if (lp->rx_head != lp->rx_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) event_mask |= LDC_EVENT_DATA_READY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) if (lp->chan_state != orig_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) while (lp->rx_head != lp->rx_tail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) unsigned long new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) switch (p->type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) case LDC_CTRL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) err = process_control_frame(lp, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) if (err > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) event_mask |= err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) case LDC_DATA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) event_mask |= LDC_EVENT_DATA_READY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) case LDC_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) err = process_error_frame(lp, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) err = LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) new = lp->rx_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) new += LDC_PACKET_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) if (new == (lp->rx_num_entries * LDC_PACKET_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) new = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) lp->rx_head = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) err = __set_rx_head(lp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) (void) LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) if (lp->hs_state == LDC_HS_COMPLETE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) goto handshake_complete;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) spin_unlock_irqrestore(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) send_events(lp, event_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) static irqreturn_t ldc_tx(int irq, void *dev_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) struct ldc_channel *lp = dev_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) unsigned long flags, orig_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) unsigned int event_mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) spin_lock_irqsave(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) orig_state = lp->chan_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) /* We should probably check for hypervisor errors here and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) * reset the LDC channel if we get one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) sun4v_ldc_tx_get_state(lp->id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) &lp->tx_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) &lp->tx_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) &lp->chan_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) orig_state, lp->chan_state, lp->tx_head, lp->tx_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) if (lp->cfg.mode == LDC_MODE_RAW &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) lp->chan_state == LDC_CHANNEL_UP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) lp->hs_state = LDC_HS_COMPLETE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) ldc_set_state(lp, LDC_STATE_CONNECTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) * Generate an LDC_EVENT_UP event if the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) * was not already up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) if (orig_state != LDC_CHANNEL_UP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) event_mask |= LDC_EVENT_UP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) orig_state = lp->chan_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) spin_unlock_irqrestore(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) send_events(lp, event_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) * XXX that addition and removal from the ldc_channel_list has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) * XXX atomicity, otherwise the __ldc_channel_exists() check is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) * XXX totally pointless as another thread can slip into ldc_alloc()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * XXX and add a channel with the same ID. There also needs to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) * XXX a spinlock for ldc_channel_list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) static HLIST_HEAD(ldc_channel_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) static int __ldc_channel_exists(unsigned long id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) struct ldc_channel *lp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) hlist_for_each_entry(lp, &ldc_channel_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) if (lp->id == id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) static int alloc_queue(const char *name, unsigned long num_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) struct ldc_packet **base, unsigned long *ra)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) unsigned long size, order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) void *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) size = num_entries * LDC_PACKET_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) order = get_order(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) q = (void *) __get_free_pages(GFP_KERNEL, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) if (!q) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) printk(KERN_ERR PFX "Alloc of %s queue failed with "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) "size=%lu order=%lu\n", name, size, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) memset(q, 0, PAGE_SIZE << order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) *base = q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) *ra = __pa(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) static void free_queue(unsigned long num_entries, struct ldc_packet *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) unsigned long size, order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) if (!q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) size = num_entries * LDC_PACKET_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) order = get_order(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) free_pages((unsigned long)q, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) cookie &= ~COOKIE_PGSZ_CODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) return (cookie >> (13ULL + (szcode * 3ULL)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) unsigned long entry, unsigned long npages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) struct ldc_mtable_entry *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) unsigned long i, shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) base = iommu->page_table + entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) for (i = 0; i < npages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) if (base->cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) sun4v_ldc_revoke(id, cookie + (i << shift),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) base->cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) base->mte = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) /* XXX Make this configurable... XXX */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) #define LDC_IOTABLE_SIZE (8 * 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) unsigned long sz, num_tsb_entries, tsbsize, order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) struct ldc_iommu *ldc_iommu = &lp->iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) struct ldc_mtable_entry *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) unsigned long hv_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) num_tsb_entries = LDC_IOTABLE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) spin_lock_init(&ldc_iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) sz = num_tsb_entries / 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) sz = (sz + 7UL) & ~7UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) iommu->map = kzalloc(sz, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) if (!iommu->map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) NULL, false /* no large pool */,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) 1 /* npools */,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) true /* skip span boundary check */);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) order = get_order(tsbsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) table = (struct ldc_mtable_entry *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) __get_free_pages(GFP_KERNEL, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) if (!table) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) printk(KERN_ERR PFX "Alloc of MTE table failed, "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) "size=%lu order=%lu\n", tsbsize, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) goto out_free_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) memset(table, 0, PAGE_SIZE << order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) ldc_iommu->page_table = table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) num_tsb_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) goto out_free_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) out_free_table:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) free_pages((unsigned long) table, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) ldc_iommu->page_table = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) out_free_map:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) kfree(iommu->map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) iommu->map = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) static void ldc_iommu_release(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) struct ldc_iommu *ldc_iommu = &lp->iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) unsigned long num_tsb_entries, tsbsize, order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) (void) sun4v_ldc_set_map_table(lp->id, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) num_tsb_entries = iommu->poolsize * iommu->nr_pools;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) order = get_order(tsbsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) free_pages((unsigned long) ldc_iommu->page_table, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) ldc_iommu->page_table = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) kfree(iommu->map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) iommu->map = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) struct ldc_channel *ldc_alloc(unsigned long id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) const struct ldc_channel_config *cfgp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) void *event_arg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) struct ldc_channel *lp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) const struct ldc_mode_ops *mops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) unsigned long dummy1, dummy2, hv_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) u8 mss, *mssbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) if (!ldom_domaining_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) if (!cfgp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) if (!name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) switch (cfgp->mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) case LDC_MODE_RAW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) mops = &raw_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) mss = LDC_PACKET_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) case LDC_MODE_UNRELIABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) mops = &nonraw_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) mss = LDC_PACKET_SIZE - 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) case LDC_MODE_STREAM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) mops = &stream_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) mss = LDC_PACKET_SIZE - 8 - 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) if (hv_err == HV_ECHANNEL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) err = -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) if (__ldc_channel_exists(id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) mssbuf = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) lp = kzalloc(sizeof(*lp), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) if (!lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) spin_lock_init(&lp->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) lp->id = id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) err = ldc_iommu_init(name, lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) goto out_free_ldc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) lp->mops = mops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) lp->mss = mss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) lp->cfg = *cfgp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) if (!lp->cfg.mtu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) lp->cfg.mtu = LDC_DEFAULT_MTU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) if (lp->cfg.mode == LDC_MODE_STREAM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) if (!mssbuf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) goto out_free_iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) lp->mssbuf = mssbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) lp->event_arg = event_arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) /* XXX allow setting via ldc_channel_config to override defaults
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) * XXX or use some formula based upon mtu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) err = alloc_queue("TX", lp->tx_num_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) &lp->tx_base, &lp->tx_ra);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) goto out_free_mssbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) err = alloc_queue("RX", lp->rx_num_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) &lp->rx_base, &lp->rx_ra);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) goto out_free_txq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) lp->flags |= LDC_FLAG_ALLOCED_QUEUES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) lp->hs_state = LDC_HS_CLOSED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) ldc_set_state(lp, LDC_STATE_INIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) INIT_HLIST_NODE(&lp->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) hlist_add_head(&lp->list, &ldc_channel_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) INIT_HLIST_HEAD(&lp->mh_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) lp->rx_irq_name, lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) goto out_free_txq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) lp->tx_irq_name, lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) free_irq(lp->cfg.rx_irq, lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) goto out_free_txq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) return lp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) out_free_txq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) free_queue(lp->tx_num_entries, lp->tx_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) out_free_mssbuf:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) kfree(mssbuf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) out_free_iommu:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) ldc_iommu_release(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) out_free_ldc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) kfree(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) out_err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) return ERR_PTR(err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) EXPORT_SYMBOL(ldc_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) void ldc_unbind(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) if (lp->flags & LDC_FLAG_REGISTERED_IRQS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) free_irq(lp->cfg.rx_irq, lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) free_irq(lp->cfg.tx_irq, lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) sun4v_ldc_tx_qconf(lp->id, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) sun4v_ldc_rx_qconf(lp->id, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) free_queue(lp->tx_num_entries, lp->tx_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) free_queue(lp->rx_num_entries, lp->rx_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) ldc_set_state(lp, LDC_STATE_INIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) EXPORT_SYMBOL(ldc_unbind);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) void ldc_free(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) ldc_unbind(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) hlist_del(&lp->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) kfree(lp->mssbuf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) ldc_iommu_release(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) kfree(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) EXPORT_SYMBOL(ldc_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) /* Bind the channel. This registers the LDC queues with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) * the hypervisor and puts the channel into a pseudo-listening
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) * state. This does not initiate a handshake, ldc_connect() does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) * that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) int ldc_bind(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) unsigned long hv_err, flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) int err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) if (lp->state != LDC_STATE_INIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) spin_lock_irqsave(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) enable_irq(lp->cfg.rx_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) enable_irq(lp->cfg.tx_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) lp->flags |= LDC_FLAG_REGISTERED_IRQS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) goto out_free_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) goto out_free_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) goto out_unmap_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) goto out_unmap_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) lp->flags |= LDC_FLAG_REGISTERED_QUEUES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) hv_err = sun4v_ldc_tx_get_state(lp->id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) &lp->tx_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) &lp->tx_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) &lp->chan_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) err = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) goto out_unmap_rx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) lp->tx_acked = lp->tx_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) lp->hs_state = LDC_HS_OPEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) ldc_set_state(lp, LDC_STATE_BOUND);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) if (lp->cfg.mode == LDC_MODE_RAW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) * There is no handshake in RAW mode, so handshake
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) * is completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) lp->hs_state = LDC_HS_COMPLETE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) spin_unlock_irqrestore(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) out_unmap_rx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) sun4v_ldc_rx_qconf(lp->id, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) out_unmap_tx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) sun4v_ldc_tx_qconf(lp->id, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) out_free_irqs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) free_irq(lp->cfg.tx_irq, lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) free_irq(lp->cfg.rx_irq, lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) spin_unlock_irqrestore(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) EXPORT_SYMBOL(ldc_bind);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) int ldc_connect(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) if (lp->cfg.mode == LDC_MODE_RAW)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) spin_lock_irqsave(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) lp->hs_state != LDC_HS_OPEN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) err = start_handshake(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) spin_unlock_irqrestore(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) EXPORT_SYMBOL(ldc_connect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) int ldc_disconnect(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) unsigned long hv_err, flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) if (lp->cfg.mode == LDC_MODE_RAW)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) !(lp->flags & LDC_FLAG_REGISTERED_QUEUES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) spin_lock_irqsave(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) ldc_set_state(lp, LDC_STATE_BOUND);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) lp->hs_state = LDC_HS_OPEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) lp->flags |= LDC_FLAG_RESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) spin_unlock_irqrestore(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) out_err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) sun4v_ldc_tx_qconf(lp->id, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) sun4v_ldc_rx_qconf(lp->id, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) free_irq(lp->cfg.tx_irq, lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) free_irq(lp->cfg.rx_irq, lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) LDC_FLAG_REGISTERED_QUEUES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) ldc_set_state(lp, LDC_STATE_INIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) spin_unlock_irqrestore(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) EXPORT_SYMBOL(ldc_disconnect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) int ldc_state(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) return lp->state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) EXPORT_SYMBOL(ldc_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) void ldc_set_state(struct ldc_channel *lp, u8 state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) ldcdbg(STATE, "STATE (%s) --> (%s)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) state_to_str(lp->state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) state_to_str(state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) lp->state = state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) EXPORT_SYMBOL(ldc_set_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) int ldc_mode(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) return lp->cfg.mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) EXPORT_SYMBOL(ldc_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) int ldc_rx_reset(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) return __set_rx_head(lp, lp->rx_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) EXPORT_SYMBOL(ldc_rx_reset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) void __ldc_print(struct ldc_channel *lp, const char *caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) "\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) "\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) "\trcv_nxt=%u snd_nxt=%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) caller, lp->id, lp->flags, state_to_str(lp->state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) lp->chan_state, lp->hs_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) lp->rx_head, lp->rx_tail, lp->rx_num_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) lp->tx_head, lp->tx_tail, lp->tx_num_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) lp->rcv_nxt, lp->snd_nxt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) EXPORT_SYMBOL(__ldc_print);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) unsigned long new_tail, hv_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) &lp->chan_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) if (unlikely(hv_err))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) if (size > LDC_PACKET_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) return -EMSGSIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) p = data_get_tx_packet(lp, &new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) memcpy(p, buf, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) err = send_tx_packet(lp, p, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) err = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) unsigned long hv_err, new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) if (size < LDC_PACKET_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) hv_err = sun4v_ldc_rx_get_state(lp->id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) &lp->rx_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) &lp->rx_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) &lp->chan_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) if (lp->chan_state == LDC_CHANNEL_DOWN ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) lp->chan_state == LDC_CHANNEL_RESETTING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) return -ECONNRESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) if (lp->rx_head == lp->rx_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) memcpy(buf, p, LDC_PACKET_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) new = rx_advance(lp, lp->rx_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) lp->rx_head = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) err = __set_rx_head(lp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) err = -ECONNRESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) err = LDC_PACKET_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) static const struct ldc_mode_ops raw_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) .write = write_raw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) .read = read_raw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) static int write_nonraw(struct ldc_channel *lp, const void *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) unsigned long hv_err, tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) unsigned int copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) u32 seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) &lp->chan_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) if (unlikely(hv_err))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) if (!tx_has_space_for(lp, size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) seq = lp->snd_nxt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) copied = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) tail = lp->tx_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) while (copied < size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) p->u.u_data :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) p->u.r.r_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) int data_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) p->type = LDC_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) p->stype = LDC_INFO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) p->ctrl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) data_len = size - copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) if (data_len > lp->mss)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) data_len = lp->mss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) BUG_ON(data_len > LDC_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) p->env = (data_len |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) (copied == 0 ? LDC_START : 0) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) (data_len == size - copied ? LDC_STOP : 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) p->seqid = ++seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) p->type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) p->stype,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) p->ctrl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) p->env,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) p->seqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) memcpy(data, buf, data_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) buf += data_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) copied += data_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) tail = tx_advance(lp, tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) err = set_tx_tail(lp, tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) lp->snd_nxt = seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) err = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) struct ldc_packet *first_frag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) if (first_frag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) lp->rcv_nxt = first_frag->seqid - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) err = send_data_nack(lp, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) err = ldc_rx_reset(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) if (p->stype & LDC_ACK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) int err = process_data_ack(lp, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) if (p->stype & LDC_NACK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) unsigned long dummy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) int limit = 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) cur_head, lp->rx_head, lp->rx_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) while (limit-- > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) unsigned long hv_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) hv_err = sun4v_ldc_rx_get_state(lp->id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) &dummy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) &lp->rx_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) &lp->chan_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) if (lp->chan_state == LDC_CHANNEL_DOWN ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) lp->chan_state == LDC_CHANNEL_RESETTING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) return -ECONNRESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) if (cur_head != lp->rx_tail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) ldcdbg(DATA, "DATA WAIT DONE "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) "head[%lx] tail[%lx] chan_state[%lx]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) dummy, lp->rx_tail, lp->chan_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) udelay(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) static int rx_set_head(struct ldc_channel *lp, unsigned long head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) int err = __set_rx_head(lp, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) lp->rx_head = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) static void send_data_ack(struct ldc_channel *lp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) unsigned long new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) p = data_get_tx_packet(lp, &new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) if (likely(p)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) memset(p, 0, sizeof(*p));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) p->type = LDC_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) p->stype = LDC_ACK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) p->ctrl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) p->seqid = lp->snd_nxt + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) p->u.r.ackid = lp->rcv_nxt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) err = send_tx_packet(lp, p, new_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) lp->snd_nxt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) struct ldc_packet *first_frag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) unsigned long hv_err, new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) int err, copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) hv_err = sun4v_ldc_rx_get_state(lp->id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) &lp->rx_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) &lp->rx_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) &lp->chan_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) return LDC_ABORT(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) if (lp->chan_state == LDC_CHANNEL_DOWN ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) lp->chan_state == LDC_CHANNEL_RESETTING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) return -ECONNRESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) if (lp->rx_head == lp->rx_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) first_frag = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) copied = err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) new = lp->rx_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) struct ldc_packet *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) int pkt_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) BUG_ON(new == lp->rx_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) p = lp->rx_base + (new / LDC_PACKET_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) "rcv_nxt[%08x]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) p->type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) p->stype,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) p->ctrl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) p->env,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) p->seqid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) p->u.r.ackid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) lp->rcv_nxt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) if (unlikely(!rx_seq_ok(lp, p->seqid))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) err = rx_bad_seq(lp, p, first_frag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) copied = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) if (p->type & LDC_CTRL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) err = process_control_frame(lp, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) lp->rcv_nxt = p->seqid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) * If this is a control-only packet, there is nothing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) * else to do but advance the rx queue since the packet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) * was already processed above.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) if (!(p->type & LDC_DATA)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) new = rx_advance(lp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) if (p->stype & (LDC_ACK | LDC_NACK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) err = data_ack_nack(lp, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) if (!(p->stype & LDC_INFO)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) new = rx_advance(lp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) err = rx_set_head(lp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) goto no_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) pkt_len = p->env & LDC_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) /* Every initial packet starts with the START bit set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) * Singleton packets will have both START+STOP set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) * Fragments will have START set in the first frame, STOP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) * set in the last frame, and neither bit set in middle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) * frames of the packet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) * Therefore if we are at the beginning of a packet and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) * we don't see START, or we are in the middle of a fragmented
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) * packet and do see START, we are unsynchronized and should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) * flush the RX queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) if ((first_frag == NULL && !(p->env & LDC_START)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) (first_frag != NULL && (p->env & LDC_START))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) if (!first_frag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) new = rx_advance(lp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) err = rx_set_head(lp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) if (!first_frag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) goto no_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) if (!first_frag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) first_frag = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) if (pkt_len > size - copied) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) /* User didn't give us a big enough buffer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) * what to do? This is a pretty serious error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) * Since we haven't updated the RX ring head to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) * consume any of the packets, signal the error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) * to the user and just leave the RX ring alone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) * This seems the best behavior because this allows
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) * a user of the LDC layer to start with a small
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) * RX buffer for ldc_read() calls and use -EMSGSIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) * as a cue to enlarge it's read buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) err = -EMSGSIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) /* Ok, we are gonna eat this one. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) new = rx_advance(lp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) memcpy(buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) (lp->cfg.mode == LDC_MODE_UNRELIABLE ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) p->u.u_data : p->u.r.r_data), pkt_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) buf += pkt_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) copied += pkt_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) if (p->env & LDC_STOP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) no_data:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) if (new == lp->rx_tail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) err = rx_data_wait(lp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) err = rx_set_head(lp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) if (err && first_frag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) lp->rcv_nxt = first_frag->seqid - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) err = copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) send_data_ack(lp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) static const struct ldc_mode_ops nonraw_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) .write = write_nonraw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) .read = read_nonraw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) static int write_stream(struct ldc_channel *lp, const void *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) if (size > lp->cfg.mtu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) size = lp->cfg.mtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) return write_nonraw(lp, buf, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) if (!lp->mssbuf_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) lp->mssbuf_len = err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) lp->mssbuf_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) if (size > lp->mssbuf_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) size = lp->mssbuf_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) memcpy(buf, lp->mssbuf + lp->mssbuf_off, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) lp->mssbuf_off += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) lp->mssbuf_len -= size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) static const struct ldc_mode_ops stream_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) .write = write_stream,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) .read = read_stream,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) if (!size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) spin_lock_irqsave(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) if (lp->hs_state != LDC_HS_COMPLETE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) err = -ENOTCONN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) err = lp->mops->write(lp, buf, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) spin_unlock_irqrestore(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) EXPORT_SYMBOL(ldc_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) ldcdbg(RX, "%s: entered size=%d\n", __func__, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) if (!size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) spin_lock_irqsave(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) if (lp->hs_state != LDC_HS_COMPLETE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) err = -ENOTCONN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) err = lp->mops->read(lp, buf, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) spin_unlock_irqrestore(&lp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) lp->cfg.mode, lp->rx_head, lp->rx_tail, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) EXPORT_SYMBOL(ldc_read);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) static u64 pagesize_code(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) switch (PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) case (8ULL * 1024ULL):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) case (64ULL * 1024ULL):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) case (512ULL * 1024ULL):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) return 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) case (4ULL * 1024ULL * 1024ULL):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) return 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) case (32ULL * 1024ULL * 1024ULL):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) return 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) case (256ULL * 1024ULL * 1024ULL):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) return 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) (index << PAGE_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) page_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) unsigned long npages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) long entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) npages, NULL, (unsigned long)-1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) if (unlikely(entry == IOMMU_ERROR_CODE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) return iommu->page_table + entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) static u64 perm_to_mte(unsigned int map_perm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) u64 mte_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) mte_base = pagesize_code();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) if (map_perm & LDC_MAP_SHADOW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) if (map_perm & LDC_MAP_R)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) mte_base |= LDC_MTE_COPY_R;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) if (map_perm & LDC_MAP_W)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) mte_base |= LDC_MTE_COPY_W;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) if (map_perm & LDC_MAP_DIRECT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) if (map_perm & LDC_MAP_R)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) mte_base |= LDC_MTE_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) if (map_perm & LDC_MAP_W)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) mte_base |= LDC_MTE_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) if (map_perm & LDC_MAP_X)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) mte_base |= LDC_MTE_EXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) if (map_perm & LDC_MAP_IO) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) if (map_perm & LDC_MAP_R)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) mte_base |= LDC_MTE_IOMMU_R;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) if (map_perm & LDC_MAP_W)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) mte_base |= LDC_MTE_IOMMU_W;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) return mte_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) static int pages_in_region(unsigned long base, long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) int count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) unsigned long new = (base + PAGE_SIZE) & PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) len -= (new - base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) base = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) } while (len > 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) struct cookie_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) struct ldc_mtable_entry *page_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) struct ldc_trans_cookie *cookies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) u64 mte_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) u64 prev_cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) u32 pte_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) u32 nc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) static void fill_cookies(struct cookie_state *sp, unsigned long pa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) unsigned long off, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) unsigned long tlen, new = pa + PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) u64 this_cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) sp->page_table[sp->pte_idx].mte = sp->mte_base | pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) tlen = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) if (off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) tlen = PAGE_SIZE - off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) if (tlen > len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) tlen = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) this_cookie = make_cookie(sp->pte_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) pagesize_code(), off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) if (this_cookie == sp->prev_cookie) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) sp->cookies[sp->nc - 1].cookie_size += tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) sp->cookies[sp->nc].cookie_addr = this_cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) sp->cookies[sp->nc].cookie_size = tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) sp->nc++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) sp->prev_cookie = this_cookie + tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) sp->pte_idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) len -= tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) pa = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) } while (len > 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) static int sg_count_one(struct scatterlist *sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) long len = sg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) if ((sg->offset | len) & (8UL - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) return pages_in_region(base + sg->offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) static int sg_count_pages(struct scatterlist *sg, int num_sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) for (i = 0; i < num_sg; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) int err = sg_count_one(sg + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) count += err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) int ldc_map_sg(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) struct scatterlist *sg, int num_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) struct ldc_trans_cookie *cookies, int ncookies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) unsigned int map_perm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) unsigned long i, npages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) struct ldc_mtable_entry *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) struct cookie_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) struct ldc_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) struct scatterlist *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) if (map_perm & ~LDC_MAP_ALL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) err = sg_count_pages(sg, num_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) npages = err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) if (err > ncookies)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) return -EMSGSIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) iommu = &lp->iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) base = alloc_npages(iommu, npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) if (!base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) state.page_table = iommu->page_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) state.cookies = cookies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) state.mte_base = perm_to_mte(map_perm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) state.prev_cookie = ~(u64)0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) state.pte_idx = (base - iommu->page_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) state.nc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) for_each_sg(sg, s, num_sg, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) s->offset, s->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) return state.nc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) EXPORT_SYMBOL(ldc_map_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) int ldc_map_single(struct ldc_channel *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) void *buf, unsigned int len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) struct ldc_trans_cookie *cookies, int ncookies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) unsigned int map_perm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) unsigned long npages, pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) struct ldc_mtable_entry *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) struct cookie_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) struct ldc_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) pa = __pa(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) if ((pa | len) & (8UL - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) npages = pages_in_region(pa, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) iommu = &lp->iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) base = alloc_npages(iommu, npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) if (!base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) state.page_table = iommu->page_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) state.cookies = cookies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) state.mte_base = perm_to_mte(map_perm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) state.prev_cookie = ~(u64)0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) state.pte_idx = (base - iommu->page_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) state.nc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) BUG_ON(state.nc > ncookies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) return state.nc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) EXPORT_SYMBOL(ldc_map_single);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) static void free_npages(unsigned long id, struct ldc_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) u64 cookie, u64 size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) unsigned long npages, entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) entry = ldc_cookie_to_index(cookie, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) ldc_demap(iommu, id, cookie, entry, npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) int ncookies)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) struct ldc_iommu *iommu = &lp->iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) spin_lock_irqsave(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) for (i = 0; i < ncookies; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) u64 addr = cookies[i].cookie_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) u64 size = cookies[i].cookie_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) free_npages(lp->id, iommu, addr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) spin_unlock_irqrestore(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) EXPORT_SYMBOL(ldc_unmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) int ldc_copy(struct ldc_channel *lp, int copy_dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) void *buf, unsigned int len, unsigned long offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) struct ldc_trans_cookie *cookies, int ncookies)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) unsigned int orig_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) unsigned long ra;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) lp->id, copy_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) ra = __pa(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) if ((ra | len | offset) & (8UL - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) "ra[%lx] len[%x] offset[%lx]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) lp->id, ra, len, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) if (lp->hs_state != LDC_HS_COMPLETE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) (lp->flags & LDC_FLAG_RESET)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) "flags[%x]\n", lp->id, lp->hs_state, lp->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) return -ECONNRESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) orig_len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) for (i = 0; i < ncookies; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) unsigned long cookie_raddr = cookies[i].cookie_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) unsigned long this_len = cookies[i].cookie_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) unsigned long actual_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) if (unlikely(offset)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) unsigned long this_off = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) if (this_off > this_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) this_off = this_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) offset -= this_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) this_len -= this_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) if (!this_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) cookie_raddr += this_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) if (this_len > len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) this_len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) unsigned long hv_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) hv_err = sun4v_ldc_copy(lp->id, copy_dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) cookie_raddr, ra,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) this_len, &actual_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) if (unlikely(hv_err)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) printk(KERN_ERR PFX "ldc_copy: ID[%lu] "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) "HV error %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) lp->id, hv_err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) if (lp->hs_state != LDC_HS_COMPLETE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) (lp->flags & LDC_FLAG_RESET))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) return -ECONNRESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) cookie_raddr += actual_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) ra += actual_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) len -= actual_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) if (actual_len == this_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) this_len -= actual_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) if (!len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) /* It is caller policy what to do about short copies.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) * For example, a networking driver can declare the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) * packet a runt and drop it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) return orig_len - len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) EXPORT_SYMBOL(ldc_copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) struct ldc_trans_cookie *cookies, int *ncookies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) unsigned int map_perm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) void *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) if (len & (8UL - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) buf = kzalloc(len, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) kfree(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) return ERR_PTR(err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) *ncookies = err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) return buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) EXPORT_SYMBOL(ldc_alloc_exp_dring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) struct ldc_trans_cookie *cookies, int ncookies)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) ldc_unmap(lp, cookies, ncookies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) kfree(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) EXPORT_SYMBOL(ldc_free_exp_dring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) static int __init ldc_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) unsigned long major, minor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) struct mdesc_handle *hp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) const u64 *v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) u64 mp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) hp = mdesc_grab();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) if (!hp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) if (mp == MDESC_NODE_NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) v = mdesc_get_property(hp, mp, "domaining-enabled", NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) if (!v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) major = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) minor = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) printk(KERN_INFO PFX "Could not register LDOM hvapi.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) printk(KERN_INFO "%s", version);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) if (!*v) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) printk(KERN_INFO PFX "Domaining disabled.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) ldom_domaining_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) mdesc_release(hp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) core_initcall(ldc_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags