^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * iSCSI over TCP/IP Data-Path lib
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2004 Dmitry Yusupov
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2004 Alex Aizman
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2005 - 2006 Mike Christie
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * maintained by open-iscsi@googlegroups.com
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Credits:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Christoph Hellwig
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * FUJITA Tomonori
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * Arne Redlich
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Zhenyu Wang
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <crypto/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/inet.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/kfifo.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/scatterlist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <net/tcp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <scsi/scsi_cmnd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <scsi/scsi_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <scsi/scsi_host.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <scsi/scsi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <scsi/scsi_transport_iscsi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <trace/events/iscsi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include "iscsi_tcp.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) MODULE_AUTHOR("Mike Christie <michaelc@cs.wisc.edu>, "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) "Dmitry Yusupov <dmitry_yus@yahoo.com>, "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) "Alex Aizman <itn780@yahoo.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) MODULE_DESCRIPTION("iSCSI/TCP data-path");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) static int iscsi_dbg_libtcp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) module_param_named(debug_libiscsi_tcp, iscsi_dbg_libtcp, int,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) S_IRUGO | S_IWUSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) MODULE_PARM_DESC(debug_libiscsi_tcp, "Turn on debugging for libiscsi_tcp "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) "module. Set to 1 to turn on, and zero to turn off. Default "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) "is off.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define ISCSI_DBG_TCP(_conn, dbg_fmt, arg...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) if (iscsi_dbg_libtcp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) iscsi_conn_printk(KERN_INFO, _conn, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) "%s " dbg_fmt, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) __func__, ##arg); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) iscsi_dbg_trace(trace_iscsi_dbg_tcp, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) &(_conn)->cls_conn->dev, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) "%s " dbg_fmt, __func__, ##arg);\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) } while (0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) static int iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct iscsi_segment *segment);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * Scatterlist handling: inside the iscsi_segment, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * remember an index into the scatterlist, and set data/size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * to the current scatterlist entry. For highmem pages, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * kmap as needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * Note that the page is unmapped when we return from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * TCP's data_ready handler, so we may end up mapping and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * unmapping the same page repeatedly. The whole reason
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * for this is that we shouldn't keep the page mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * outside the softirq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * iscsi_tcp_segment_init_sg - init indicated scatterlist entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * @segment: the buffer object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * @sg: scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * @offset: byte offset into that sg entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * This function sets up the segment so that subsequent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * data is copied to the indicated sg entry, at the given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) iscsi_tcp_segment_init_sg(struct iscsi_segment *segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) struct scatterlist *sg, unsigned int offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) segment->sg = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) segment->sg_offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) segment->size = min(sg->length - offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) segment->total_size - segment->total_copied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) segment->data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * iscsi_tcp_segment_map - map the current S/G page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * @segment: iscsi_segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * @recv: 1 if called from recv path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * We only need to possibly kmap data if scatter lists are being used,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * because the iscsi passthrough and internal IO paths will never use high
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * mem pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static void iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (segment->data != NULL || !segment->sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) sg = segment->sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) BUG_ON(segment->sg_mapped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) BUG_ON(sg->length == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * We always map for the recv path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * If the page count is greater than one it is ok to send
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * to the network layer's zero copy send path. If not we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * have to go the slow sendmsg path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * Same goes for slab pages: skb_can_coalesce() allows
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * coalescing neighboring slab objects into a single frag which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * triggers one of hardened usercopy checks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (!recv && sendpage_ok(sg_page(sg)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) if (recv) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) segment->atomic_mapped = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) segment->sg_mapped = kmap_atomic(sg_page(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) segment->atomic_mapped = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) /* the xmit path can sleep with the page mapped so use kmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) segment->sg_mapped = kmap(sg_page(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) segment->data = segment->sg_mapped + sg->offset + segment->sg_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) void iscsi_tcp_segment_unmap(struct iscsi_segment *segment)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (segment->sg_mapped) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) if (segment->atomic_mapped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) kunmap_atomic(segment->sg_mapped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) kunmap(sg_page(segment->sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) segment->sg_mapped = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) segment->data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) EXPORT_SYMBOL_GPL(iscsi_tcp_segment_unmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * Splice the digest buffer into the buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) segment->data = digest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) segment->digest_len = ISCSI_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) segment->total_size += ISCSI_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) segment->size = ISCSI_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) segment->copied = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) segment->sg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) segment->hash = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * iscsi_tcp_segment_done - check whether the segment is complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * @tcp_conn: iscsi tcp connection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * @segment: iscsi segment to check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * @recv: set to one of this is called from the recv path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * @copied: number of bytes copied
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * Check if we're done receiving this segment. If the receive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * buffer is full but we expect more data, move on to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * next entry in the scatterlist.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * If the amount of data we received isn't a multiple of 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * we will transparently receive the pad bytes, too.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * This function must be re-entrant.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) int iscsi_tcp_segment_done(struct iscsi_tcp_conn *tcp_conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) struct iscsi_segment *segment, int recv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) unsigned copied)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) struct scatterlist sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) unsigned int pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copied %u %u size %u %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) segment->copied, copied, segment->size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) recv ? "recv" : "xmit");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (segment->hash && copied) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * If a segment is kmapd we must unmap it before sending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * to the crypto layer since that will try to kmap it again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) iscsi_tcp_segment_unmap(segment);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (!segment->data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) sg_init_table(&sg, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) sg_set_page(&sg, sg_page(segment->sg), copied,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) segment->copied + segment->sg_offset +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) segment->sg->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) sg_init_one(&sg, segment->data + segment->copied,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) copied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) ahash_request_set_crypt(segment->hash, &sg, NULL, copied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) crypto_ahash_update(segment->hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) segment->copied += copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (segment->copied < segment->size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) iscsi_tcp_segment_map(segment, recv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) segment->total_copied += segment->copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) segment->copied = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) segment->size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /* Unmap the current scatterlist page, if there is one. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) iscsi_tcp_segment_unmap(segment);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) /* Do we have more scatterlist entries? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "total copied %u total size %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) segment->total_copied, segment->total_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (segment->total_copied < segment->total_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /* Proceed to the next entry in the scatterlist. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) iscsi_tcp_segment_map(segment, recv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) BUG_ON(segment->size == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) /* Do we need to handle padding? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (!(tcp_conn->iscsi_conn->session->tt->caps & CAP_PADDING_OFFLOAD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) pad = iscsi_padding(segment->total_copied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if (pad != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) ISCSI_DBG_TCP(tcp_conn->iscsi_conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) "consume %d pad bytes\n", pad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) segment->total_size += pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) segment->size = pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) segment->data = segment->padbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * Set us up for transferring the data digest. hdr digest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * is completely handled in hdr done function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (segment->hash) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) ahash_request_set_crypt(segment->hash, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) segment->digest, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) crypto_ahash_final(segment->hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) iscsi_tcp_segment_splice_digest(segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) recv ? segment->recv_digest : segment->digest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) EXPORT_SYMBOL_GPL(iscsi_tcp_segment_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * iscsi_tcp_segment_recv - copy data to segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * @tcp_conn: the iSCSI TCP connection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * @segment: the buffer to copy to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * @ptr: data pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * @len: amount of data available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * This function copies up to @len bytes to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * given buffer, and returns the number of bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) * consumed, which can actually be less than @len.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * If hash digest is enabled, the function will update the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * hash while copying.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * Combining these two operations doesn't buy us a lot (yet),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * but in the future we could implement combined copy+crc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * just way we do for network layer checksums.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) iscsi_tcp_segment_recv(struct iscsi_tcp_conn *tcp_conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) struct iscsi_segment *segment, const void *ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) unsigned int copy = 0, copied = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) while (!iscsi_tcp_segment_done(tcp_conn, segment, 1, copy)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) if (copied == len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) ISCSI_DBG_TCP(tcp_conn->iscsi_conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) "copied %d bytes\n", len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) copy = min(len - copied, segment->size - segment->copied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copying %d\n", copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) memcpy(segment->data + segment->copied, ptr + copied, copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) copied += copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) return copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) iscsi_tcp_dgst_header(struct ahash_request *hash, const void *hdr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) size_t hdrlen, unsigned char digest[ISCSI_DIGEST_SIZE])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) struct scatterlist sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) sg_init_one(&sg, hdr, hdrlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) ahash_request_set_crypt(hash, &sg, digest, hdrlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) crypto_ahash_digest(hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) EXPORT_SYMBOL_GPL(iscsi_tcp_dgst_header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) static inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) iscsi_tcp_dgst_verify(struct iscsi_tcp_conn *tcp_conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) struct iscsi_segment *segment)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (!segment->digest_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (memcmp(segment->recv_digest, segment->digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) segment->digest_len)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "digest mismatch\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * Helper function to set up segment buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) __iscsi_segment_init(struct iscsi_segment *segment, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) iscsi_segment_done_fn_t *done, struct ahash_request *hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) memset(segment, 0, sizeof(*segment));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) segment->total_size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) segment->done = done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (hash) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) segment->hash = hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) crypto_ahash_init(hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) iscsi_segment_init_linear(struct iscsi_segment *segment, void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) size_t size, iscsi_segment_done_fn_t *done,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) struct ahash_request *hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) __iscsi_segment_init(segment, size, done, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) segment->data = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) segment->size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) EXPORT_SYMBOL_GPL(iscsi_segment_init_linear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) iscsi_segment_seek_sg(struct iscsi_segment *segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) struct scatterlist *sg_list, unsigned int sg_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) unsigned int offset, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) iscsi_segment_done_fn_t *done,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) struct ahash_request *hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) __iscsi_segment_init(segment, size, done, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) for_each_sg(sg_list, sg, sg_count, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) if (offset < sg->length) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) iscsi_tcp_segment_init_sg(segment, sg, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) offset -= sg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) return ISCSI_ERR_DATA_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) EXPORT_SYMBOL_GPL(iscsi_segment_seek_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * @tcp_conn: iscsi connection to prep for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) * This function always passes NULL for the hash argument, because when this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) * function is called we do not yet know the final size of the header and want
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) * to delay the digest processing until we know that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) void iscsi_tcp_hdr_recv_prep(struct iscsi_tcp_conn *tcp_conn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) ISCSI_DBG_TCP(tcp_conn->iscsi_conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) "(%s)\n", tcp_conn->iscsi_conn->hdrdgst_en ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) "digest enabled" : "digest disabled");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) iscsi_segment_init_linear(&tcp_conn->in.segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) tcp_conn->in.hdr_buf, sizeof(struct iscsi_hdr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) iscsi_tcp_hdr_recv_done, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) EXPORT_SYMBOL_GPL(iscsi_tcp_hdr_recv_prep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * Handle incoming reply to any other type of command
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) iscsi_tcp_data_recv_done(struct iscsi_tcp_conn *tcp_conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) struct iscsi_segment *segment)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) struct iscsi_conn *conn = tcp_conn->iscsi_conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) return ISCSI_ERR_DATA_DGST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) conn->data, tcp_conn->in.datalen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) iscsi_tcp_hdr_recv_prep(tcp_conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) iscsi_tcp_data_recv_prep(struct iscsi_tcp_conn *tcp_conn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) struct iscsi_conn *conn = tcp_conn->iscsi_conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) struct ahash_request *rx_hash = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (conn->datadgst_en &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) rx_hash = tcp_conn->rx_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) iscsi_segment_init_linear(&tcp_conn->in.segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) conn->data, tcp_conn->in.datalen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) iscsi_tcp_data_recv_done, rx_hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * iscsi_tcp_cleanup_task - free tcp_task resources
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * @task: iscsi task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * must be called with session back_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) void iscsi_tcp_cleanup_task(struct iscsi_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) struct iscsi_tcp_task *tcp_task = task->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) struct iscsi_r2t_info *r2t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) /* nothing to do for mgmt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (!task->sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) spin_lock_bh(&tcp_task->queue2pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /* flush task's r2t queues */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) while (kfifo_out(&tcp_task->r2tqueue, (void*)&r2t, sizeof(void*))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) kfifo_in(&tcp_task->r2tpool.queue, (void*)&r2t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) sizeof(void*));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) ISCSI_DBG_TCP(task->conn, "pending r2t dropped\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) r2t = tcp_task->r2t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) if (r2t != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) kfifo_in(&tcp_task->r2tpool.queue, (void*)&r2t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) sizeof(void*));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) tcp_task->r2t = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) spin_unlock_bh(&tcp_task->queue2pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) EXPORT_SYMBOL_GPL(iscsi_tcp_cleanup_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * iscsi_tcp_data_in - SCSI Data-In Response processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) * @conn: iscsi connection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) * @task: scsi command task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) static int iscsi_tcp_data_in(struct iscsi_conn *conn, struct iscsi_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) struct iscsi_tcp_task *tcp_task = task->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) int datasn = be32_to_cpu(rhdr->datasn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) unsigned total_in_length = task->sc->sdb.length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) * lib iscsi will update this in the completion handling if there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) * is status.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) if (!(rhdr->flags & ISCSI_FLAG_DATA_STATUS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) iscsi_update_cmdsn(conn->session, (struct iscsi_nopin*)rhdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (tcp_conn->in.datalen == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) if (tcp_task->exp_datasn != datasn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) ISCSI_DBG_TCP(conn, "task->exp_datasn(%d) != rhdr->datasn(%d)"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) "\n", tcp_task->exp_datasn, datasn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) return ISCSI_ERR_DATASN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) tcp_task->exp_datasn++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) tcp_task->data_offset = be32_to_cpu(rhdr->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (tcp_task->data_offset + tcp_conn->in.datalen > total_in_length) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) ISCSI_DBG_TCP(conn, "data_offset(%d) + data_len(%d) > "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) "total_length_in(%d)\n", tcp_task->data_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) tcp_conn->in.datalen, total_in_length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return ISCSI_ERR_DATA_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) conn->datain_pdus_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * iscsi_tcp_r2t_rsp - iSCSI R2T Response processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * @conn: iscsi connection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) * @task: scsi command task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) static int iscsi_tcp_r2t_rsp(struct iscsi_conn *conn, struct iscsi_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) struct iscsi_session *session = conn->session;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) struct iscsi_tcp_task *tcp_task = task->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) struct iscsi_r2t_info *r2t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) int r2tsn = be32_to_cpu(rhdr->r2tsn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) u32 data_length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) u32 data_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) if (tcp_conn->in.datalen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) iscsi_conn_printk(KERN_ERR, conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) "invalid R2t with datalen %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) tcp_conn->in.datalen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) return ISCSI_ERR_DATALEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (tcp_task->exp_datasn != r2tsn){
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) ISCSI_DBG_TCP(conn, "task->exp_datasn(%d) != rhdr->r2tsn(%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) tcp_task->exp_datasn, r2tsn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) return ISCSI_ERR_R2TSN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) /* fill-in new R2T associated with the task */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) iscsi_update_cmdsn(session, (struct iscsi_nopin*)rhdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) if (!task->sc || session->state != ISCSI_STATE_LOGGED_IN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) iscsi_conn_printk(KERN_INFO, conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) "dropping R2T itt %d in recovery.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) task->itt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) data_length = be32_to_cpu(rhdr->data_length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if (data_length == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) iscsi_conn_printk(KERN_ERR, conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) "invalid R2T with zero data len\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) return ISCSI_ERR_DATALEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) if (data_length > session->max_burst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) ISCSI_DBG_TCP(conn, "invalid R2T with data len %u and max "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) "burst %u. Attempting to execute request.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) data_length, session->max_burst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) data_offset = be32_to_cpu(rhdr->data_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) if (data_offset + data_length > task->sc->sdb.length) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) iscsi_conn_printk(KERN_ERR, conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) "invalid R2T with data len %u at offset %u "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) "and total length %d\n", data_length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) data_offset, task->sc->sdb.length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) return ISCSI_ERR_DATALEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) spin_lock(&tcp_task->pool2queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) rc = kfifo_out(&tcp_task->r2tpool.queue, (void *)&r2t, sizeof(void *));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (!rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) iscsi_conn_printk(KERN_ERR, conn, "Could not allocate R2T. "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) "Target has sent more R2Ts than it "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) "negotiated for or driver has leaked.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) spin_unlock(&tcp_task->pool2queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) return ISCSI_ERR_PROTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) r2t->exp_statsn = rhdr->statsn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) r2t->data_length = data_length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) r2t->data_offset = data_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) r2t->ttt = rhdr->ttt; /* no flip */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) r2t->datasn = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) r2t->sent = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) tcp_task->exp_datasn = r2tsn + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) kfifo_in(&tcp_task->r2tqueue, (void*)&r2t, sizeof(void*));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) conn->r2t_pdus_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) spin_unlock(&tcp_task->pool2queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) iscsi_requeue_task(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) * Handle incoming reply to DataIn command
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) iscsi_tcp_process_data_in(struct iscsi_tcp_conn *tcp_conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) struct iscsi_segment *segment)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) struct iscsi_conn *conn = tcp_conn->iscsi_conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) struct iscsi_hdr *hdr = tcp_conn->in.hdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) return ISCSI_ERR_DATA_DGST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) /* check for non-exceptional status */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) iscsi_tcp_hdr_recv_prep(tcp_conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * iscsi_tcp_hdr_dissect - process PDU header
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) * @conn: iSCSI connection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) * @hdr: PDU header
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) * This function analyzes the header of the PDU received,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) * and performs several sanity checks. If the PDU is accompanied
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) * by data, the receive buffer is set up to copy the incoming data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) * to the correct location.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) int rc = 0, opcode, ahslen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) struct iscsi_task *task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) /* verify PDU length */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) tcp_conn->in.datalen = ntoh24(hdr->dlength);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) if (tcp_conn->in.datalen > conn->max_recv_dlength) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) iscsi_conn_printk(KERN_ERR, conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) "iscsi_tcp: datalen %d > %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) tcp_conn->in.datalen, conn->max_recv_dlength);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) return ISCSI_ERR_DATALEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) /* Additional header segments. So far, we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * process additional headers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) ahslen = hdr->hlength << 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) opcode = hdr->opcode & ISCSI_OPCODE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) /* verify itt (itt encoding: age+cid+itt) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) rc = iscsi_verify_itt(conn, hdr->itt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) ISCSI_DBG_TCP(conn, "opcode 0x%x ahslen %d datalen %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) opcode, ahslen, tcp_conn->in.datalen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) switch(opcode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) case ISCSI_OP_SCSI_DATA_IN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) spin_lock(&conn->session->back_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) task = iscsi_itt_to_ctask(conn, hdr->itt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) if (!task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) rc = ISCSI_ERR_BAD_ITT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) rc = iscsi_tcp_data_in(conn, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) spin_unlock(&conn->session->back_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) if (tcp_conn->in.datalen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) struct iscsi_tcp_task *tcp_task = task->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) struct ahash_request *rx_hash = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) struct scsi_data_buffer *sdb = &task->sc->sdb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * Setup copy of Data-In into the struct scsi_cmnd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * Scatterlist case:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * We set up the iscsi_segment to point to the next
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) * scatterlist entry to copy to. As we go along,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * we move on to the next scatterlist entry and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * update the digest per-entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) if (conn->datadgst_en &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) rx_hash = tcp_conn->rx_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) ISCSI_DBG_TCP(conn, "iscsi_tcp_begin_data_in( "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) "offset=%d, datalen=%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) tcp_task->data_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) tcp_conn->in.datalen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) task->last_xfer = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) rc = iscsi_segment_seek_sg(&tcp_conn->in.segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) sdb->table.sgl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) sdb->table.nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) tcp_task->data_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) tcp_conn->in.datalen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) iscsi_tcp_process_data_in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) rx_hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) spin_unlock(&conn->session->back_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) spin_unlock(&conn->session->back_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) case ISCSI_OP_SCSI_CMD_RSP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (tcp_conn->in.datalen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) iscsi_tcp_data_recv_prep(tcp_conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) case ISCSI_OP_R2T:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) spin_lock(&conn->session->back_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) task = iscsi_itt_to_ctask(conn, hdr->itt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) spin_unlock(&conn->session->back_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (!task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) rc = ISCSI_ERR_BAD_ITT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) else if (ahslen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) rc = ISCSI_ERR_AHSLEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) else if (task->sc->sc_data_direction == DMA_TO_DEVICE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) task->last_xfer = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) spin_lock(&conn->session->frwd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) rc = iscsi_tcp_r2t_rsp(conn, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) spin_unlock(&conn->session->frwd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) rc = ISCSI_ERR_PROTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) case ISCSI_OP_LOGIN_RSP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) case ISCSI_OP_TEXT_RSP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) case ISCSI_OP_REJECT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) case ISCSI_OP_ASYNC_EVENT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * It is possible that we could get a PDU with a buffer larger
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * than 8K, but there are no targets that currently do this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) * For now we fail until we find a vendor that needs it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) if (ISCSI_DEF_MAX_RECV_SEG_LEN < tcp_conn->in.datalen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) iscsi_conn_printk(KERN_ERR, conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) "iscsi_tcp: received buffer of "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) "len %u but conn buffer is only %u "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) "(opcode %0x)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) tcp_conn->in.datalen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) ISCSI_DEF_MAX_RECV_SEG_LEN, opcode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) rc = ISCSI_ERR_PROTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) /* If there's data coming in with the response,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * receive it to the connection's buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if (tcp_conn->in.datalen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) iscsi_tcp_data_recv_prep(tcp_conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) case ISCSI_OP_LOGOUT_RSP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) case ISCSI_OP_NOOP_IN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) case ISCSI_OP_SCSI_TMFUNC_RSP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) rc = ISCSI_ERR_BAD_OPCODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) if (rc == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) /* Anything that comes with data should have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) * been handled above. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) if (tcp_conn->in.datalen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) return ISCSI_ERR_PROTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) iscsi_tcp_hdr_recv_prep(tcp_conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) * iscsi_tcp_hdr_recv_done - process PDU header
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * @tcp_conn: iSCSI TCP connection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * @segment: the buffer segment being processed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) * This is the callback invoked when the PDU header has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * been received. If the header is followed by additional
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) * header segments, we go back for more data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) struct iscsi_segment *segment)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) struct iscsi_conn *conn = tcp_conn->iscsi_conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) struct iscsi_hdr *hdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) /* Check if there are additional header segments
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * *prior* to computing the digest, because we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * may need to go back to the caller for more.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) hdr = (struct iscsi_hdr *) tcp_conn->in.hdr_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) /* Bump the header length - the caller will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) * just loop around and get the AHS for us, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) * call again. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) unsigned int ahslen = hdr->hlength << 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) /* Make sure we don't overflow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) return ISCSI_ERR_AHSLEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) segment->total_size += ahslen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) segment->size += ahslen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) /* We're done processing the header. See if we're doing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * header digests; if so, set up the recv_digest buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) * and go back for more. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) if (conn->hdrdgst_en &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) if (segment->digest_len == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) * Even if we offload the digest processing we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) * splice it in so we can increment the skb/segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * counters in preparation for the data segment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) iscsi_tcp_segment_splice_digest(segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) segment->recv_digest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) iscsi_tcp_dgst_header(tcp_conn->rx_hash, hdr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) segment->total_copied - ISCSI_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) segment->digest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) return ISCSI_ERR_HDR_DGST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) tcp_conn->in.hdr = hdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) return iscsi_tcp_hdr_dissect(conn, hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) * iscsi_tcp_recv_segment_is_hdr - tests if we are reading in a header
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) * @tcp_conn: iscsi tcp conn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * returns non zero if we are currently processing or setup to process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) * a header.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) inline int iscsi_tcp_recv_segment_is_hdr(struct iscsi_tcp_conn *tcp_conn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) EXPORT_SYMBOL_GPL(iscsi_tcp_recv_segment_is_hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * iscsi_tcp_recv_skb - Process skb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * @conn: iscsi connection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * @skb: network buffer with header and/or data segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * @offset: offset in skb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * @offloaded: bool indicating if transfer was offloaded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * @status: iscsi TCP status result
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) * Will return status of transfer in @status. And will return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) * number of bytes copied.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) int iscsi_tcp_recv_skb(struct iscsi_conn *conn, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) unsigned int offset, bool offloaded, int *status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) struct iscsi_segment *segment = &tcp_conn->in.segment;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) struct skb_seq_state seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) unsigned int consumed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) ISCSI_DBG_TCP(conn, "in %d bytes\n", skb->len - offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * Update for each skb instead of pdu, because over slow networks a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * data_in's data could take a while to read in. We also want to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * account for r2ts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) conn->last_recv = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) if (unlikely(conn->suspend_rx)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) ISCSI_DBG_TCP(conn, "Rx suspended!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) *status = ISCSI_TCP_SUSPENDED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) if (offloaded) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) segment->total_copied = segment->total_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) goto segment_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) skb_prepare_seq_read(skb, offset, skb->len, &seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) unsigned int avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) const u8 *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) avail = skb_seq_read(consumed, &ptr, &seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) if (avail == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) ISCSI_DBG_TCP(conn, "no more data avail. Consumed %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) consumed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) *status = ISCSI_TCP_SKB_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) goto skb_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) BUG_ON(segment->copied >= segment->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) ISCSI_DBG_TCP(conn, "skb %p ptr=%p avail=%u\n", skb, ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) rc = iscsi_tcp_segment_recv(tcp_conn, segment, ptr, avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) BUG_ON(rc == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) consumed += rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) if (segment->total_copied >= segment->total_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) skb_abort_seq_read(&seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) goto segment_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) segment_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) *status = ISCSI_TCP_SEGMENT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) ISCSI_DBG_TCP(conn, "segment done\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) rc = segment->done(tcp_conn, segment);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) if (rc != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) *status = ISCSI_TCP_CONN_ERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) ISCSI_DBG_TCP(conn, "Error receiving PDU, errno=%d\n", rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) iscsi_conn_failure(conn, rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) /* The done() functions sets up the next segment. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) skb_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) conn->rxdata_octets += consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) return consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) EXPORT_SYMBOL_GPL(iscsi_tcp_recv_skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) * iscsi_tcp_task_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * @task: scsi command task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) int iscsi_tcp_task_init(struct iscsi_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) struct iscsi_tcp_task *tcp_task = task->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) struct iscsi_conn *conn = task->conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) struct scsi_cmnd *sc = task->sc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) if (!sc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) * mgmt tasks do not have a scatterlist since they come
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) * in from the iscsi interface.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) ISCSI_DBG_TCP(conn, "mtask deq [itt 0x%x]\n", task->itt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) return conn->session->tt->init_pdu(task, 0, task->data_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) BUG_ON(kfifo_len(&tcp_task->r2tqueue));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) tcp_task->exp_datasn = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) /* Prepare PDU, optionally w/ immediate data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) ISCSI_DBG_TCP(conn, "task deq [itt 0x%x imm %d unsol %d]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) task->itt, task->imm_count, task->unsol_r2t.data_length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) err = conn->session->tt->init_pdu(task, 0, task->imm_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) task->imm_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) EXPORT_SYMBOL_GPL(iscsi_tcp_task_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) static struct iscsi_r2t_info *iscsi_tcp_get_curr_r2t(struct iscsi_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) struct iscsi_tcp_task *tcp_task = task->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) struct iscsi_r2t_info *r2t = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) if (iscsi_task_has_unsol_data(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) r2t = &task->unsol_r2t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) spin_lock_bh(&tcp_task->queue2pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) if (tcp_task->r2t) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) r2t = tcp_task->r2t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) /* Continue with this R2T? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) if (r2t->data_length <= r2t->sent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) ISCSI_DBG_TCP(task->conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) " done with r2t %p\n", r2t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) kfifo_in(&tcp_task->r2tpool.queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) (void *)&tcp_task->r2t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) sizeof(void *));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) tcp_task->r2t = r2t = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) if (r2t == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) if (kfifo_out(&tcp_task->r2tqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) (void *)&tcp_task->r2t, sizeof(void *)) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) sizeof(void *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) r2t = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) r2t = tcp_task->r2t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) spin_unlock_bh(&tcp_task->queue2pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) return r2t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) * iscsi_tcp_task_xmit - xmit normal PDU task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) * @task: iscsi command task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) * We're expected to return 0 when everything was transmitted successfully,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) * -EAGAIN if there's still data in the queue, or != 0 for any other kind
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) * of error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) int iscsi_tcp_task_xmit(struct iscsi_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) struct iscsi_conn *conn = task->conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) struct iscsi_session *session = conn->session;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) struct iscsi_r2t_info *r2t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) flush:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) /* Flush any pending data first. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) rc = session->tt->xmit_pdu(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) /* mgmt command */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) if (!task->sc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) if (task->hdr->itt == RESERVED_ITT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) iscsi_put_task(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) /* Are we done already? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) if (task->sc->sc_data_direction != DMA_TO_DEVICE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) r2t = iscsi_tcp_get_curr_r2t(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) if (r2t == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) /* Waiting for more R2Ts to arrive. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) ISCSI_DBG_TCP(conn, "no R2Ts yet\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) rc = conn->session->tt->alloc_pdu(task, ISCSI_OP_SCSI_DATA_OUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) iscsi_prep_data_out_pdu(task, r2t, (struct iscsi_data *) task->hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) ISCSI_DBG_TCP(conn, "sol dout %p [dsn %d itt 0x%x doff %d dlen %d]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) r2t, r2t->datasn - 1, task->hdr->itt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) r2t->data_offset + r2t->sent, r2t->data_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) rc = conn->session->tt->init_pdu(task, r2t->data_offset + r2t->sent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) r2t->data_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) iscsi_conn_failure(conn, ISCSI_ERR_XMIT_FAILED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) r2t->sent += r2t->data_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) goto flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) EXPORT_SYMBOL_GPL(iscsi_tcp_task_xmit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) struct iscsi_cls_conn *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) iscsi_tcp_conn_setup(struct iscsi_cls_session *cls_session, int dd_data_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) uint32_t conn_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) struct iscsi_conn *conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) struct iscsi_cls_conn *cls_conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) struct iscsi_tcp_conn *tcp_conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) cls_conn = iscsi_conn_setup(cls_session,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) sizeof(*tcp_conn) + dd_data_size, conn_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) if (!cls_conn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) conn = cls_conn->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) * due to strange issues with iser these are not set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) * in iscsi_conn_setup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) tcp_conn = conn->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) tcp_conn->iscsi_conn = conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) tcp_conn->dd_data = conn->dd_data + sizeof(*tcp_conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) return cls_conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) EXPORT_SYMBOL_GPL(iscsi_tcp_conn_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) void iscsi_tcp_conn_teardown(struct iscsi_cls_conn *cls_conn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) iscsi_conn_teardown(cls_conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) EXPORT_SYMBOL_GPL(iscsi_tcp_conn_teardown);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) int iscsi_tcp_r2tpool_alloc(struct iscsi_session *session)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) int cmd_i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) * initialize per-task: R2T pool and xmit queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) struct iscsi_task *task = session->cmds[cmd_i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) struct iscsi_tcp_task *tcp_task = task->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) * pre-allocated x2 as much r2ts to handle race when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) * target acks DataOut faster than we data_xmit() queues
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) * could replenish r2tqueue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) /* R2T pool */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) if (iscsi_pool_init(&tcp_task->r2tpool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) session->max_r2t * 2, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) sizeof(struct iscsi_r2t_info))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) goto r2t_alloc_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) /* R2T xmit queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) if (kfifo_alloc(&tcp_task->r2tqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) session->max_r2t * 4 * sizeof(void*), GFP_KERNEL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) iscsi_pool_free(&tcp_task->r2tpool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) goto r2t_alloc_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) spin_lock_init(&tcp_task->pool2queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) spin_lock_init(&tcp_task->queue2pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) r2t_alloc_fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) for (i = 0; i < cmd_i; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) struct iscsi_task *task = session->cmds[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) struct iscsi_tcp_task *tcp_task = task->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) kfifo_free(&tcp_task->r2tqueue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) iscsi_pool_free(&tcp_task->r2tpool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) EXPORT_SYMBOL_GPL(iscsi_tcp_r2tpool_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) void iscsi_tcp_r2tpool_free(struct iscsi_session *session)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) for (i = 0; i < session->cmds_max; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) struct iscsi_task *task = session->cmds[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) struct iscsi_tcp_task *tcp_task = task->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) kfifo_free(&tcp_task->r2tqueue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) iscsi_pool_free(&tcp_task->r2tpool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) EXPORT_SYMBOL_GPL(iscsi_tcp_r2tpool_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) int iscsi_tcp_set_max_r2t(struct iscsi_conn *conn, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) struct iscsi_session *session = conn->session;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) unsigned short r2ts = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) sscanf(buf, "%hu", &r2ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) if (session->max_r2t == r2ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) if (!r2ts || !is_power_of_2(r2ts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) session->max_r2t = r2ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) iscsi_tcp_r2tpool_free(session);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) return iscsi_tcp_r2tpool_alloc(session);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) EXPORT_SYMBOL_GPL(iscsi_tcp_set_max_r2t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) void iscsi_tcp_conn_get_stats(struct iscsi_cls_conn *cls_conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) struct iscsi_stats *stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) struct iscsi_conn *conn = cls_conn->dd_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) stats->txdata_octets = conn->txdata_octets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) stats->rxdata_octets = conn->rxdata_octets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) stats->dataout_pdus = conn->dataout_pdus_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) stats->datain_pdus = conn->datain_pdus_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) stats->r2t_pdus = conn->r2t_pdus_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) EXPORT_SYMBOL_GPL(iscsi_tcp_conn_get_stats);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags