1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* NVMe over Fabrics TCP target.
4	* Copyright (c) 2018 Lightbits Labs. All rights reserved.
5	*/
6	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7	#include <linux/module.h>
8	#include <linux/init.h>
9	#include <linux/slab.h>
10	#include <linux/err.h>
11	#include <linux/key.h>
12	#include <linux/nvme-tcp.h>
13	#include <linux/nvme-keyring.h>
14	#include <net/sock.h>
15	#include <net/tcp.h>
16	#include <net/tls.h>
17	#include <net/tls_prot.h>
18	#include <net/handshake.h>
19	#include <linux/inet.h>
20	#include <linux/llist.h>
21	#include <crypto/hash.h>
22	#include <trace/events/sock.h>
23
24	#include "nvmet.h"
25
26	#define NVMET_TCP_DEF_INLINE_DATA_SIZE (4 * PAGE_SIZE)
27	#define NVMET_TCP_MAXH2CDATA 0x400000 /* 16M arbitrary limit */
28	#define NVMET_TCP_BACKLOG 128
29
30	static int param_store_val(const char *str, int *val, int min, int max)
31	{
32	int ret, new_val;
33
34	ret = kstrtoint(s: str, base: 10, res: &new_val);
35	if (ret)
36	return -EINVAL;
37
38	if (new_val < min || new_val > max)
39	return -EINVAL;
40
41	*val = new_val;
42	return 0;
43	}
44
45	static int set_params(const char *str, const struct kernel_param *kp)
46	{
47	return param_store_val(str, val: kp->arg, min: 0, INT_MAX);
48	}
49
50	static const struct kernel_param_ops set_param_ops = {
51	.set = set_params,
52	.get = param_get_int,
53	};
54
55	/* Define the socket priority to use for connections were it is desirable
56	* that the NIC consider performing optimized packet processing or filtering.
57	* A non-zero value being sufficient to indicate general consideration of any
58	* possible optimization. Making it a module param allows for alternative
59	* values that may be unique for some NIC implementations.
60	*/
61	static int so_priority;
62	device_param_cb(so_priority, &set_param_ops, &so_priority, 0644);
63	MODULE_PARM_DESC(so_priority, "nvmet tcp socket optimize priority: Default 0");
64
65	/* Define a time period (in usecs) that io_work() shall sample an activated
66	* queue before determining it to be idle. This optional module behavior
67	* can enable NIC solutions that support socket optimized packet processing
68	* using advanced interrupt moderation techniques.
69	*/
70	static int idle_poll_period_usecs;
71	device_param_cb(idle_poll_period_usecs, &set_param_ops,
72	&idle_poll_period_usecs, 0644);
73	MODULE_PARM_DESC(idle_poll_period_usecs,
74	"nvmet tcp io_work poll till idle time period in usecs: Default 0");
75
76	#ifdef CONFIG_NVME_TARGET_TCP_TLS
77	/*
78	* TLS handshake timeout
79	*/
80	static int tls_handshake_timeout = 10;
81	module_param(tls_handshake_timeout, int, 0644);
82	MODULE_PARM_DESC(tls_handshake_timeout,
83	"nvme TLS handshake timeout in seconds (default 10)");
84	#endif
85
86	#define NVMET_TCP_RECV_BUDGET 8
87	#define NVMET_TCP_SEND_BUDGET 8
88	#define NVMET_TCP_IO_WORK_BUDGET 64
89
90	enum nvmet_tcp_send_state {
91	NVMET_TCP_SEND_DATA_PDU,
92	NVMET_TCP_SEND_DATA,
93	NVMET_TCP_SEND_R2T,
94	NVMET_TCP_SEND_DDGST,
95	NVMET_TCP_SEND_RESPONSE
96	};
97
98	enum nvmet_tcp_recv_state {
99	NVMET_TCP_RECV_PDU,
100	NVMET_TCP_RECV_DATA,
101	NVMET_TCP_RECV_DDGST,
102	NVMET_TCP_RECV_ERR,
103	};
104
105	enum {
106	NVMET_TCP_F_INIT_FAILED = (1 << 0),
107	};
108
109	struct nvmet_tcp_cmd {
110	struct nvmet_tcp_queue *queue;
111	struct nvmet_req req;
112
113	struct nvme_tcp_cmd_pdu *cmd_pdu;
114	struct nvme_tcp_rsp_pdu *rsp_pdu;
115	struct nvme_tcp_data_pdu *data_pdu;
116	struct nvme_tcp_r2t_pdu *r2t_pdu;
117
118	u32 rbytes_done;
119	u32 wbytes_done;
120
121	u32 pdu_len;
122	u32 pdu_recv;
123	int sg_idx;
124	char recv_cbuf[CMSG_LEN(sizeof(char))];
125	struct msghdr recv_msg;
126	struct bio_vec *iov;
127	u32 flags;
128
129	struct list_head entry;
130	struct llist_node lentry;
131
132	/* send state */
133	u32 offset;
134	struct scatterlist *cur_sg;
135	enum nvmet_tcp_send_state state;
136
137	__le32 exp_ddgst;
138	__le32 recv_ddgst;
139	};
140
141	enum nvmet_tcp_queue_state {
142	NVMET_TCP_Q_CONNECTING,
143	NVMET_TCP_Q_TLS_HANDSHAKE,
144	NVMET_TCP_Q_LIVE,
145	NVMET_TCP_Q_DISCONNECTING,
146	NVMET_TCP_Q_FAILED,
147	};
148
149	struct nvmet_tcp_queue {
150	struct socket *sock;
151	struct nvmet_tcp_port *port;
152	struct work_struct io_work;
153	struct nvmet_cq nvme_cq;
154	struct nvmet_sq nvme_sq;
155	struct kref kref;
156
157	/* send state */
158	struct nvmet_tcp_cmd *cmds;
159	unsigned int nr_cmds;
160	struct list_head free_list;
161	struct llist_head resp_list;
162	struct list_head resp_send_list;
163	int send_list_len;
164	struct nvmet_tcp_cmd *snd_cmd;
165
166	/* recv state */
167	int offset;
168	int left;
169	enum nvmet_tcp_recv_state rcv_state;
170	struct nvmet_tcp_cmd *cmd;
171	union nvme_tcp_pdu pdu;
172
173	/* digest state */
174	bool hdr_digest;
175	bool data_digest;
176	struct ahash_request *snd_hash;
177	struct ahash_request *rcv_hash;
178
179	/* TLS state */
180	key_serial_t tls_pskid;
181	struct delayed_work tls_handshake_tmo_work;
182
183	unsigned long poll_end;
184
185	spinlock_t state_lock;
186	enum nvmet_tcp_queue_state state;
187
188	struct sockaddr_storage sockaddr;
189	struct sockaddr_storage sockaddr_peer;
190	struct work_struct release_work;
191
192	int idx;
193	struct list_head queue_list;
194
195	struct nvmet_tcp_cmd connect;
196
197	struct page_frag_cache pf_cache;
198
199	void (*data_ready)(struct sock *);
200	void (*state_change)(struct sock *);
201	void (*write_space)(struct sock *);
202	};
203
204	struct nvmet_tcp_port {
205	struct socket *sock;
206	struct work_struct accept_work;
207	struct nvmet_port *nport;
208	struct sockaddr_storage addr;
209	void (*data_ready)(struct sock *);
210	};
211
212	static DEFINE_IDA(nvmet_tcp_queue_ida);
213	static LIST_HEAD(nvmet_tcp_queue_list);
214	static DEFINE_MUTEX(nvmet_tcp_queue_mutex);
215
216	static struct workqueue_struct *nvmet_tcp_wq;
217	static const struct nvmet_fabrics_ops nvmet_tcp_ops;
218	static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c);
219	static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd);
220
221	static inline u16 nvmet_tcp_cmd_tag(struct nvmet_tcp_queue *queue,
222	struct nvmet_tcp_cmd *cmd)
223	{
224	if (unlikely(!queue->nr_cmds)) {
225	/* We didn't allocate cmds yet, send 0xffff */
226	return USHRT_MAX;
227	}
228
229	return cmd - queue->cmds;
230	}
231
232	static inline bool nvmet_tcp_has_data_in(struct nvmet_tcp_cmd *cmd)
233	{
234	return nvme_is_write(cmd: cmd->req.cmd) &&
235	cmd->rbytes_done < cmd->req.transfer_len;
236	}
237
238	static inline bool nvmet_tcp_need_data_in(struct nvmet_tcp_cmd *cmd)
239	{
240	return nvmet_tcp_has_data_in(cmd) && !cmd->req.cqe->status;
241	}
242
243	static inline bool nvmet_tcp_need_data_out(struct nvmet_tcp_cmd *cmd)
244	{
245	return !nvme_is_write(cmd: cmd->req.cmd) &&
246	cmd->req.transfer_len > 0 &&
247	!cmd->req.cqe->status;
248	}
249
250	static inline bool nvmet_tcp_has_inline_data(struct nvmet_tcp_cmd *cmd)
251	{
252	return nvme_is_write(cmd: cmd->req.cmd) && cmd->pdu_len &&
253	!cmd->rbytes_done;
254	}
255
256	static inline struct nvmet_tcp_cmd *
257	nvmet_tcp_get_cmd(struct nvmet_tcp_queue *queue)
258	{
259	struct nvmet_tcp_cmd *cmd;
260
261	cmd = list_first_entry_or_null(&queue->free_list,
262	struct nvmet_tcp_cmd, entry);
263	if (!cmd)
264	return NULL;
265	list_del_init(entry: &cmd->entry);
266
267	cmd->rbytes_done = cmd->wbytes_done = 0;
268	cmd->pdu_len = 0;
269	cmd->pdu_recv = 0;
270	cmd->iov = NULL;
271	cmd->flags = 0;
272	return cmd;
273	}
274
275	static inline void nvmet_tcp_put_cmd(struct nvmet_tcp_cmd *cmd)
276	{
277	if (unlikely(cmd == &cmd->queue->connect))
278	return;
279
280	list_add_tail(new: &cmd->entry, head: &cmd->queue->free_list);
281	}
282
283	static inline int queue_cpu(struct nvmet_tcp_queue *queue)
284	{
285	return queue->sock->sk->sk_incoming_cpu;
286	}
287
288	static inline u8 nvmet_tcp_hdgst_len(struct nvmet_tcp_queue *queue)
289	{
290	return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
291	}
292
293	static inline u8 nvmet_tcp_ddgst_len(struct nvmet_tcp_queue *queue)
294	{
295	return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
296	}
297
298	static inline void nvmet_tcp_hdgst(struct ahash_request *hash,
299	void *pdu, size_t len)
300	{
301	struct scatterlist sg;
302
303	sg_init_one(&sg, pdu, len);
304	ahash_request_set_crypt(req: hash, src: &sg, result: pdu + len, nbytes: len);
305	crypto_ahash_digest(req: hash);
306	}
307
308	static int nvmet_tcp_verify_hdgst(struct nvmet_tcp_queue *queue,
309	void *pdu, size_t len)
310	{
311	struct nvme_tcp_hdr *hdr = pdu;
312	__le32 recv_digest;
313	__le32 exp_digest;
314
315	if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
316	pr_err("queue %d: header digest enabled but no header digest\n",
317	queue->idx);
318	return -EPROTO;
319	}
320
321	recv_digest = *(__le32 *)(pdu + hdr->hlen);
322	nvmet_tcp_hdgst(hash: queue->rcv_hash, pdu, len);
323	exp_digest = *(__le32 *)(pdu + hdr->hlen);
324	if (recv_digest != exp_digest) {
325	pr_err("queue %d: header digest error: recv %#x expected %#x\n",
326	queue->idx, le32_to_cpu(recv_digest),
327	le32_to_cpu(exp_digest));
328	return -EPROTO;
329	}
330
331	return 0;
332	}
333
334	static int nvmet_tcp_check_ddgst(struct nvmet_tcp_queue *queue, void *pdu)
335	{
336	struct nvme_tcp_hdr *hdr = pdu;
337	u8 digest_len = nvmet_tcp_hdgst_len(queue);
338	u32 len;
339
340	len = le32_to_cpu(hdr->plen) - hdr->hlen -
341	(hdr->flags & NVME_TCP_F_HDGST ? digest_len : 0);
342
343	if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
344	pr_err("queue %d: data digest flag is cleared\n", queue->idx);
345	return -EPROTO;
346	}
347
348	return 0;
349	}
350
351	static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd)
352	{
353	kfree(objp: cmd->iov);
354	sgl_free(sgl: cmd->req.sg);
355	cmd->iov = NULL;
356	cmd->req.sg = NULL;
357	}
358
359	static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd)
360	{
361	struct bio_vec *iov = cmd->iov;
362	struct scatterlist *sg;
363	u32 length, offset, sg_offset;
364	int nr_pages;
365
366	length = cmd->pdu_len;
367	nr_pages = DIV_ROUND_UP(length, PAGE_SIZE);
368	offset = cmd->rbytes_done;
369	cmd->sg_idx = offset / PAGE_SIZE;
370	sg_offset = offset % PAGE_SIZE;
371	sg = &cmd->req.sg[cmd->sg_idx];
372
373	while (length) {
374	u32 iov_len = min_t(u32, length, sg->length - sg_offset);
375
376	bvec_set_page(bv: iov, page: sg_page(sg), len: iov_len,
377	offset: sg->offset + sg_offset);
378
379	length -= iov_len;
380	sg = sg_next(sg);
381	iov++;
382	sg_offset = 0;
383	}
384
385	iov_iter_bvec(i: &cmd->recv_msg.msg_iter, ITER_DEST, bvec: cmd->iov,
386	nr_segs: nr_pages, count: cmd->pdu_len);
387	}
388
389	static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue)
390	{
391	queue->rcv_state = NVMET_TCP_RECV_ERR;
392	if (queue->nvme_sq.ctrl)
393	nvmet_ctrl_fatal_error(ctrl: queue->nvme_sq.ctrl);
394	else
395	kernel_sock_shutdown(sock: queue->sock, how: SHUT_RDWR);
396	}
397
398	static void nvmet_tcp_socket_error(struct nvmet_tcp_queue *queue, int status)
399	{
400	queue->rcv_state = NVMET_TCP_RECV_ERR;
401	if (status == -EPIPE || status == -ECONNRESET)
402	kernel_sock_shutdown(sock: queue->sock, how: SHUT_RDWR);
403	else
404	nvmet_tcp_fatal_error(queue);
405	}
406
407	static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd)
408	{
409	struct nvme_sgl_desc *sgl = &cmd->req.cmd->common.dptr.sgl;
410	u32 len = le32_to_cpu(sgl->length);
411
412	if (!len)
413	return 0;
414
415	if (sgl->type == ((NVME_SGL_FMT_DATA_DESC << 4) |
416	NVME_SGL_FMT_OFFSET)) {
417	if (!nvme_is_write(cmd: cmd->req.cmd))
418	return NVME_SC_INVALID_FIELD | NVME_SC_DNR;
419
420	if (len > cmd->req.port->inline_data_size)
421	return NVME_SC_SGL_INVALID_OFFSET | NVME_SC_DNR;
422	cmd->pdu_len = len;
423	}
424	cmd->req.transfer_len += len;
425
426	cmd->req.sg = sgl_alloc(length: len, GFP_KERNEL, nent_p: &cmd->req.sg_cnt);
427	if (!cmd->req.sg)
428	return NVME_SC_INTERNAL;
429	cmd->cur_sg = cmd->req.sg;
430
431	if (nvmet_tcp_has_data_in(cmd)) {
432	cmd->iov = kmalloc_array(n: cmd->req.sg_cnt,
433	size: sizeof(*cmd->iov), GFP_KERNEL);
434	if (!cmd->iov)
435	goto err;
436	}
437
438	return 0;
439	err:
440	nvmet_tcp_free_cmd_buffers(cmd);
441	return NVME_SC_INTERNAL;
442	}
443
444	static void nvmet_tcp_calc_ddgst(struct ahash_request *hash,
445	struct nvmet_tcp_cmd *cmd)
446	{
447	ahash_request_set_crypt(req: hash, src: cmd->req.sg,
448	result: (void *)&cmd->exp_ddgst, nbytes: cmd->req.transfer_len);
449	crypto_ahash_digest(req: hash);
450	}
451
452	static void nvmet_setup_c2h_data_pdu(struct nvmet_tcp_cmd *cmd)
453	{
454	struct nvme_tcp_data_pdu *pdu = cmd->data_pdu;
455	struct nvmet_tcp_queue *queue = cmd->queue;
456	u8 hdgst = nvmet_tcp_hdgst_len(queue: cmd->queue);
457	u8 ddgst = nvmet_tcp_ddgst_len(queue: cmd->queue);
458
459	cmd->offset = 0;
460	cmd->state = NVMET_TCP_SEND_DATA_PDU;
461
462	pdu->hdr.type = nvme_tcp_c2h_data;
463	pdu->hdr.flags = NVME_TCP_F_DATA_LAST | (queue->nvme_sq.sqhd_disabled ?
464	NVME_TCP_F_DATA_SUCCESS : 0);
465	pdu->hdr.hlen = sizeof(*pdu);
466	pdu->hdr.pdo = pdu->hdr.hlen + hdgst;
467	pdu->hdr.plen =
468	cpu_to_le32(pdu->hdr.hlen + hdgst +
469	cmd->req.transfer_len + ddgst);
470	pdu->command_id = cmd->req.cqe->command_id;
471	pdu->data_length = cpu_to_le32(cmd->req.transfer_len);
472	pdu->data_offset = cpu_to_le32(cmd->wbytes_done);
473
474	if (queue->data_digest) {
475	pdu->hdr.flags |= NVME_TCP_F_DDGST;
476	nvmet_tcp_calc_ddgst(hash: queue->snd_hash, cmd);
477	}
478
479	if (cmd->queue->hdr_digest) {
480	pdu->hdr.flags |= NVME_TCP_F_HDGST;
481	nvmet_tcp_hdgst(hash: queue->snd_hash, pdu, len: sizeof(*pdu));
482	}
483	}
484
485	static void nvmet_setup_r2t_pdu(struct nvmet_tcp_cmd *cmd)
486	{
487	struct nvme_tcp_r2t_pdu *pdu = cmd->r2t_pdu;
488	struct nvmet_tcp_queue *queue = cmd->queue;
489	u8 hdgst = nvmet_tcp_hdgst_len(queue: cmd->queue);
490
491	cmd->offset = 0;
492	cmd->state = NVMET_TCP_SEND_R2T;
493
494	pdu->hdr.type = nvme_tcp_r2t;
495	pdu->hdr.flags = 0;
496	pdu->hdr.hlen = sizeof(*pdu);
497	pdu->hdr.pdo = 0;
498	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
499
500	pdu->command_id = cmd->req.cmd->common.command_id;
501	pdu->ttag = nvmet_tcp_cmd_tag(queue: cmd->queue, cmd);
502	pdu->r2t_length = cpu_to_le32(cmd->req.transfer_len - cmd->rbytes_done);
503	pdu->r2t_offset = cpu_to_le32(cmd->rbytes_done);
504	if (cmd->queue->hdr_digest) {
505	pdu->hdr.flags |= NVME_TCP_F_HDGST;
506	nvmet_tcp_hdgst(hash: queue->snd_hash, pdu, len: sizeof(*pdu));
507	}
508	}
509
510	static void nvmet_setup_response_pdu(struct nvmet_tcp_cmd *cmd)
511	{
512	struct nvme_tcp_rsp_pdu *pdu = cmd->rsp_pdu;
513	struct nvmet_tcp_queue *queue = cmd->queue;
514	u8 hdgst = nvmet_tcp_hdgst_len(queue: cmd->queue);
515
516	cmd->offset = 0;
517	cmd->state = NVMET_TCP_SEND_RESPONSE;
518
519	pdu->hdr.type = nvme_tcp_rsp;
520	pdu->hdr.flags = 0;
521	pdu->hdr.hlen = sizeof(*pdu);
522	pdu->hdr.pdo = 0;
523	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
524	if (cmd->queue->hdr_digest) {
525	pdu->hdr.flags |= NVME_TCP_F_HDGST;
526	nvmet_tcp_hdgst(hash: queue->snd_hash, pdu, len: sizeof(*pdu));
527	}
528	}
529
530	static void nvmet_tcp_process_resp_list(struct nvmet_tcp_queue *queue)
531	{
532	struct llist_node *node;
533	struct nvmet_tcp_cmd *cmd;
534
535	for (node = llist_del_all(head: &queue->resp_list); node; node = node->next) {
536	cmd = llist_entry(node, struct nvmet_tcp_cmd, lentry);
537	list_add(new: &cmd->entry, head: &queue->resp_send_list);
538	queue->send_list_len++;
539	}
540	}
541
542	static struct nvmet_tcp_cmd *nvmet_tcp_fetch_cmd(struct nvmet_tcp_queue *queue)
543	{
544	queue->snd_cmd = list_first_entry_or_null(&queue->resp_send_list,
545	struct nvmet_tcp_cmd, entry);
546	if (!queue->snd_cmd) {
547	nvmet_tcp_process_resp_list(queue);
548	queue->snd_cmd =
549	list_first_entry_or_null(&queue->resp_send_list,
550	struct nvmet_tcp_cmd, entry);
551	if (unlikely(!queue->snd_cmd))
552	return NULL;
553	}
554
555	list_del_init(entry: &queue->snd_cmd->entry);
556	queue->send_list_len--;
557
558	if (nvmet_tcp_need_data_out(cmd: queue->snd_cmd))
559	nvmet_setup_c2h_data_pdu(cmd: queue->snd_cmd);
560	else if (nvmet_tcp_need_data_in(cmd: queue->snd_cmd))
561	nvmet_setup_r2t_pdu(cmd: queue->snd_cmd);
562	else
563	nvmet_setup_response_pdu(cmd: queue->snd_cmd);
564
565	return queue->snd_cmd;
566	}
567
568	static void nvmet_tcp_queue_response(struct nvmet_req *req)
569	{
570	struct nvmet_tcp_cmd *cmd =
571	container_of(req, struct nvmet_tcp_cmd, req);
572	struct nvmet_tcp_queue *queue = cmd->queue;
573	struct nvme_sgl_desc *sgl;
574	u32 len;
575
576	if (unlikely(cmd == queue->cmd)) {
577	sgl = &cmd->req.cmd->common.dptr.sgl;
578	len = le32_to_cpu(sgl->length);
579
580	/*
581	* Wait for inline data before processing the response.
582	* Avoid using helpers, this might happen before
583	* nvmet_req_init is completed.
584	*/
585	if (queue->rcv_state == NVMET_TCP_RECV_PDU &&
586	len && len <= cmd->req.port->inline_data_size &&
587	nvme_is_write(cmd: cmd->req.cmd))
588	return;
589	}
590
591	llist_add(new: &cmd->lentry, head: &queue->resp_list);
592	queue_work_on(cpu: queue_cpu(queue), wq: nvmet_tcp_wq, work: &cmd->queue->io_work);
593	}
594
595	static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd)
596	{
597	if (unlikely(cmd->flags & NVMET_TCP_F_INIT_FAILED))
598	nvmet_tcp_queue_response(req: &cmd->req);
599	else
600	cmd->req.execute(&cmd->req);
601	}
602
603	static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd)
604	{
605	struct msghdr msg = {
606	.msg_flags = MSG_DONTWAIT | MSG_MORE | MSG_SPLICE_PAGES,
607	};
608	struct bio_vec bvec;
609	u8 hdgst = nvmet_tcp_hdgst_len(queue: cmd->queue);
610	int left = sizeof(*cmd->data_pdu) - cmd->offset + hdgst;
611	int ret;
612
613	bvec_set_virt(bv: &bvec, vaddr: (void *)cmd->data_pdu + cmd->offset, len: left);
614	iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: &bvec, nr_segs: 1, count: left);
615	ret = sock_sendmsg(sock: cmd->queue->sock, msg: &msg);
616	if (ret <= 0)
617	return ret;
618
619	cmd->offset += ret;
620	left -= ret;
621
622	if (left)
623	return -EAGAIN;
624
625	cmd->state = NVMET_TCP_SEND_DATA;
626	cmd->offset = 0;
627	return 1;
628	}
629
630	static int nvmet_try_send_data(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
631	{
632	struct nvmet_tcp_queue *queue = cmd->queue;
633	int ret;
634
635	while (cmd->cur_sg) {
636	struct msghdr msg = {
637	.msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES,
638	};
639	struct page *page = sg_page(sg: cmd->cur_sg);
640	struct bio_vec bvec;
641	u32 left = cmd->cur_sg->length - cmd->offset;
642
643	if ((!last_in_batch && cmd->queue->send_list_len) ||
644	cmd->wbytes_done + left < cmd->req.transfer_len ||
645	queue->data_digest || !queue->nvme_sq.sqhd_disabled)
646	msg.msg_flags |= MSG_MORE;
647
648	bvec_set_page(bv: &bvec, page, len: left, offset: cmd->offset);
649	iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: &bvec, nr_segs: 1, count: left);
650	ret = sock_sendmsg(sock: cmd->queue->sock, msg: &msg);
651	if (ret <= 0)
652	return ret;
653
654	cmd->offset += ret;
655	cmd->wbytes_done += ret;
656
657	/* Done with sg?*/
658	if (cmd->offset == cmd->cur_sg->length) {
659	cmd->cur_sg = sg_next(cmd->cur_sg);
660	cmd->offset = 0;
661	}
662	}
663
664	if (queue->data_digest) {
665	cmd->state = NVMET_TCP_SEND_DDGST;
666	cmd->offset = 0;
667	} else {
668	if (queue->nvme_sq.sqhd_disabled) {
669	cmd->queue->snd_cmd = NULL;
670	nvmet_tcp_put_cmd(cmd);
671	} else {
672	nvmet_setup_response_pdu(cmd);
673	}
674	}
675
676	if (queue->nvme_sq.sqhd_disabled)
677	nvmet_tcp_free_cmd_buffers(cmd);
678
679	return 1;
680
681	}
682
683	static int nvmet_try_send_response(struct nvmet_tcp_cmd *cmd,
684	bool last_in_batch)
685	{
686	struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, };
687	struct bio_vec bvec;
688	u8 hdgst = nvmet_tcp_hdgst_len(queue: cmd->queue);
689	int left = sizeof(*cmd->rsp_pdu) - cmd->offset + hdgst;
690	int ret;
691
692	if (!last_in_batch && cmd->queue->send_list_len)
693	msg.msg_flags |= MSG_MORE;
694	else
695	msg.msg_flags |= MSG_EOR;
696
697	bvec_set_virt(bv: &bvec, vaddr: (void *)cmd->rsp_pdu + cmd->offset, len: left);
698	iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: &bvec, nr_segs: 1, count: left);
699	ret = sock_sendmsg(sock: cmd->queue->sock, msg: &msg);
700	if (ret <= 0)
701	return ret;
702	cmd->offset += ret;
703	left -= ret;
704
705	if (left)
706	return -EAGAIN;
707
708	nvmet_tcp_free_cmd_buffers(cmd);
709	cmd->queue->snd_cmd = NULL;
710	nvmet_tcp_put_cmd(cmd);
711	return 1;
712	}
713
714	static int nvmet_try_send_r2t(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
715	{
716	struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, };
717	struct bio_vec bvec;
718	u8 hdgst = nvmet_tcp_hdgst_len(queue: cmd->queue);
719	int left = sizeof(*cmd->r2t_pdu) - cmd->offset + hdgst;
720	int ret;
721
722	if (!last_in_batch && cmd->queue->send_list_len)
723	msg.msg_flags |= MSG_MORE;
724	else
725	msg.msg_flags |= MSG_EOR;
726
727	bvec_set_virt(bv: &bvec, vaddr: (void *)cmd->r2t_pdu + cmd->offset, len: left);
728	iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: &bvec, nr_segs: 1, count: left);
729	ret = sock_sendmsg(sock: cmd->queue->sock, msg: &msg);
730	if (ret <= 0)
731	return ret;
732	cmd->offset += ret;
733	left -= ret;
734
735	if (left)
736	return -EAGAIN;
737
738	cmd->queue->snd_cmd = NULL;
739	return 1;
740	}
741
742	static int nvmet_try_send_ddgst(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
743	{
744	struct nvmet_tcp_queue *queue = cmd->queue;
745	int left = NVME_TCP_DIGEST_LENGTH - cmd->offset;
746	struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
747	struct kvec iov = {
748	.iov_base = (u8 *)&cmd->exp_ddgst + cmd->offset,
749	.iov_len = left
750	};
751	int ret;
752
753	if (!last_in_batch && cmd->queue->send_list_len)
754	msg.msg_flags |= MSG_MORE;
755	else
756	msg.msg_flags |= MSG_EOR;
757
758	ret = kernel_sendmsg(sock: queue->sock, msg: &msg, vec: &iov, num: 1, len: iov.iov_len);
759	if (unlikely(ret <= 0))
760	return ret;
761
762	cmd->offset += ret;
763	left -= ret;
764
765	if (left)
766	return -EAGAIN;
767
768	if (queue->nvme_sq.sqhd_disabled) {
769	cmd->queue->snd_cmd = NULL;
770	nvmet_tcp_put_cmd(cmd);
771	} else {
772	nvmet_setup_response_pdu(cmd);
773	}
774	return 1;
775	}
776
777	static int nvmet_tcp_try_send_one(struct nvmet_tcp_queue *queue,
778	bool last_in_batch)
779	{
780	struct nvmet_tcp_cmd *cmd = queue->snd_cmd;
781	int ret = 0;
782
783	if (!cmd || queue->state == NVMET_TCP_Q_DISCONNECTING) {
784	cmd = nvmet_tcp_fetch_cmd(queue);
785	if (unlikely(!cmd))
786	return 0;
787	}
788
789	if (cmd->state == NVMET_TCP_SEND_DATA_PDU) {
790	ret = nvmet_try_send_data_pdu(cmd);
791	if (ret <= 0)
792	goto done_send;
793	}
794
795	if (cmd->state == NVMET_TCP_SEND_DATA) {
796	ret = nvmet_try_send_data(cmd, last_in_batch);
797	if (ret <= 0)
798	goto done_send;
799	}
800
801	if (cmd->state == NVMET_TCP_SEND_DDGST) {
802	ret = nvmet_try_send_ddgst(cmd, last_in_batch);
803	if (ret <= 0)
804	goto done_send;
805	}
806
807	if (cmd->state == NVMET_TCP_SEND_R2T) {
808	ret = nvmet_try_send_r2t(cmd, last_in_batch);
809	if (ret <= 0)
810	goto done_send;
811	}
812
813	if (cmd->state == NVMET_TCP_SEND_RESPONSE)
814	ret = nvmet_try_send_response(cmd, last_in_batch);
815
816	done_send:
817	if (ret < 0) {
818	if (ret == -EAGAIN)
819	return 0;
820	return ret;
821	}
822
823	return 1;
824	}
825
826	static int nvmet_tcp_try_send(struct nvmet_tcp_queue *queue,
827	int budget, int *sends)
828	{
829	int i, ret = 0;
830
831	for (i = 0; i < budget; i++) {
832	ret = nvmet_tcp_try_send_one(queue, last_in_batch: i == budget - 1);
833	if (unlikely(ret < 0)) {
834	nvmet_tcp_socket_error(queue, status: ret);
835	goto done;
836	} else if (ret == 0) {
837	break;
838	}
839	(*sends)++;
840	}
841	done:
842	return ret;
843	}
844
845	static void nvmet_prepare_receive_pdu(struct nvmet_tcp_queue *queue)
846	{
847	queue->offset = 0;
848	queue->left = sizeof(struct nvme_tcp_hdr);
849	queue->cmd = NULL;
850	queue->rcv_state = NVMET_TCP_RECV_PDU;
851	}
852
853	static void nvmet_tcp_free_crypto(struct nvmet_tcp_queue *queue)
854	{
855	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: queue->rcv_hash);
856
857	ahash_request_free(req: queue->rcv_hash);
858	ahash_request_free(req: queue->snd_hash);
859	crypto_free_ahash(tfm);
860	}
861
862	static int nvmet_tcp_alloc_crypto(struct nvmet_tcp_queue *queue)
863	{
864	struct crypto_ahash *tfm;
865
866	tfm = crypto_alloc_ahash(alg_name: "crc32c", type: 0, CRYPTO_ALG_ASYNC);
867	if (IS_ERR(ptr: tfm))
868	return PTR_ERR(ptr: tfm);
869
870	queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
871	if (!queue->snd_hash)
872	goto free_tfm;
873	ahash_request_set_callback(req: queue->snd_hash, flags: 0, NULL, NULL);
874
875	queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
876	if (!queue->rcv_hash)
877	goto free_snd_hash;
878	ahash_request_set_callback(req: queue->rcv_hash, flags: 0, NULL, NULL);
879
880	return 0;
881	free_snd_hash:
882	ahash_request_free(req: queue->snd_hash);
883	free_tfm:
884	crypto_free_ahash(tfm);
885	return -ENOMEM;
886	}
887
888
889	static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue)
890	{
891	struct nvme_tcp_icreq_pdu *icreq = &queue->pdu.icreq;
892	struct nvme_tcp_icresp_pdu *icresp = &queue->pdu.icresp;
893	struct msghdr msg = {};
894	struct kvec iov;
895	int ret;
896
897	if (le32_to_cpu(icreq->hdr.plen) != sizeof(struct nvme_tcp_icreq_pdu)) {
898	pr_err("bad nvme-tcp pdu length (%d)\n",
899	le32_to_cpu(icreq->hdr.plen));
900	nvmet_tcp_fatal_error(queue);
901	return -EPROTO;
902	}
903
904	if (icreq->pfv != NVME_TCP_PFV_1_0) {
905	pr_err("queue %d: bad pfv %d\n", queue->idx, icreq->pfv);
906	return -EPROTO;
907	}
908
909	if (icreq->hpda != 0) {
910	pr_err("queue %d: unsupported hpda %d\n", queue->idx,
911	icreq->hpda);
912	return -EPROTO;
913	}
914
915	queue->hdr_digest = !!(icreq->digest & NVME_TCP_HDR_DIGEST_ENABLE);
916	queue->data_digest = !!(icreq->digest & NVME_TCP_DATA_DIGEST_ENABLE);
917	if (queue->hdr_digest || queue->data_digest) {
918	ret = nvmet_tcp_alloc_crypto(queue);
919	if (ret)
920	return ret;
921	}
922
923	memset(icresp, 0, sizeof(*icresp));
924	icresp->hdr.type = nvme_tcp_icresp;
925	icresp->hdr.hlen = sizeof(*icresp);
926	icresp->hdr.pdo = 0;
927	icresp->hdr.plen = cpu_to_le32(icresp->hdr.hlen);
928	icresp->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
929	icresp->maxdata = cpu_to_le32(NVMET_TCP_MAXH2CDATA);
930	icresp->cpda = 0;
931	if (queue->hdr_digest)
932	icresp->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
933	if (queue->data_digest)
934	icresp->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
935
936	iov.iov_base = icresp;
937	iov.iov_len = sizeof(*icresp);
938	ret = kernel_sendmsg(sock: queue->sock, msg: &msg, vec: &iov, num: 1, len: iov.iov_len);
939	if (ret < 0) {
940	queue->state = NVMET_TCP_Q_FAILED;
941	return ret; /* queue removal will cleanup */
942	}
943
944	queue->state = NVMET_TCP_Q_LIVE;
945	nvmet_prepare_receive_pdu(queue);
946	return 0;
947	}
948
949	static void nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue,
950	struct nvmet_tcp_cmd *cmd, struct nvmet_req *req)
951	{
952	size_t data_len = le32_to_cpu(req->cmd->common.dptr.sgl.length);
953	int ret;
954
955	/*
956	* This command has not been processed yet, hence we are trying to
957	* figure out if there is still pending data left to receive. If
958	* we don't, we can simply prepare for the next pdu and bail out,
959	* otherwise we will need to prepare a buffer and receive the
960	* stale data before continuing forward.
961	*/
962	if (!nvme_is_write(cmd: cmd->req.cmd) || !data_len ||
963	data_len > cmd->req.port->inline_data_size) {
964	nvmet_prepare_receive_pdu(queue);
965	return;
966	}
967
968	ret = nvmet_tcp_map_data(cmd);
969	if (unlikely(ret)) {
970	pr_err("queue %d: failed to map data\n", queue->idx);
971	nvmet_tcp_fatal_error(queue);
972	return;
973	}
974
975	queue->rcv_state = NVMET_TCP_RECV_DATA;
976	nvmet_tcp_build_pdu_iovec(cmd);
977	cmd->flags |= NVMET_TCP_F_INIT_FAILED;
978	}
979
980	static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue)
981	{
982	struct nvme_tcp_data_pdu *data = &queue->pdu.data;
983	struct nvmet_tcp_cmd *cmd;
984	unsigned int exp_data_len;
985
986	if (likely(queue->nr_cmds)) {
987	if (unlikely(data->ttag >= queue->nr_cmds)) {
988	pr_err("queue %d: received out of bound ttag %u, nr_cmds %u\n",
989	queue->idx, data->ttag, queue->nr_cmds);
990	goto err_proto;
991	}
992	cmd = &queue->cmds[data->ttag];
993	} else {
994	cmd = &queue->connect;
995	}
996
997	if (le32_to_cpu(data->data_offset) != cmd->rbytes_done) {
998	pr_err("ttag %u unexpected data offset %u (expected %u)\n",
999	data->ttag, le32_to_cpu(data->data_offset),
1000	cmd->rbytes_done);
1001	goto err_proto;
1002	}
1003
1004	exp_data_len = le32_to_cpu(data->hdr.plen) -
1005	nvmet_tcp_hdgst_len(queue) -
1006	nvmet_tcp_ddgst_len(queue) -
1007	sizeof(*data);
1008
1009	cmd->pdu_len = le32_to_cpu(data->data_length);
1010	if (unlikely(cmd->pdu_len != exp_data_len ||
1011	cmd->pdu_len == 0 ||
1012	cmd->pdu_len > NVMET_TCP_MAXH2CDATA)) {
1013	pr_err("H2CData PDU len %u is invalid\n", cmd->pdu_len);
1014	goto err_proto;
1015	}
1016	cmd->pdu_recv = 0;
1017	nvmet_tcp_build_pdu_iovec(cmd);
1018	queue->cmd = cmd;
1019	queue->rcv_state = NVMET_TCP_RECV_DATA;
1020
1021	return 0;
1022
1023	err_proto:
1024	/* FIXME: use proper transport errors */
1025	nvmet_tcp_fatal_error(queue);
1026	return -EPROTO;
1027	}
1028
1029	static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
1030	{
1031	struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;
1032	struct nvme_command *nvme_cmd = &queue->pdu.cmd.cmd;
1033	struct nvmet_req *req;
1034	int ret;
1035
1036	if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) {
1037	if (hdr->type != nvme_tcp_icreq) {
1038	pr_err("unexpected pdu type (%d) before icreq\n",
1039	hdr->type);
1040	nvmet_tcp_fatal_error(queue);
1041	return -EPROTO;
1042	}
1043	return nvmet_tcp_handle_icreq(queue);
1044	}
1045
1046	if (unlikely(hdr->type == nvme_tcp_icreq)) {
1047	pr_err("queue %d: received icreq pdu in state %d\n",
1048	queue->idx, queue->state);
1049	nvmet_tcp_fatal_error(queue);
1050	return -EPROTO;
1051	}
1052
1053	if (hdr->type == nvme_tcp_h2c_data) {
1054	ret = nvmet_tcp_handle_h2c_data_pdu(queue);
1055	if (unlikely(ret))
1056	return ret;
1057	return 0;
1058	}
1059
1060	queue->cmd = nvmet_tcp_get_cmd(queue);
1061	if (unlikely(!queue->cmd)) {
1062	/* This should never happen */
1063	pr_err("queue %d: out of commands (%d) send_list_len: %d, opcode: %d",
1064	queue->idx, queue->nr_cmds, queue->send_list_len,
1065	nvme_cmd->common.opcode);
1066	nvmet_tcp_fatal_error(queue);
1067	return -ENOMEM;
1068	}
1069
1070	req = &queue->cmd->req;
1071	memcpy(req->cmd, nvme_cmd, sizeof(*nvme_cmd));
1072
1073	if (unlikely(!nvmet_req_init(req, &queue->nvme_cq,
1074	&queue->nvme_sq, &nvmet_tcp_ops))) {
1075	pr_err("failed cmd %p id %d opcode %d, data_len: %d\n",
1076	req->cmd, req->cmd->common.command_id,
1077	req->cmd->common.opcode,
1078	le32_to_cpu(req->cmd->common.dptr.sgl.length));
1079
1080	nvmet_tcp_handle_req_failure(queue, cmd: queue->cmd, req);
1081	return 0;
1082	}
1083
1084	ret = nvmet_tcp_map_data(cmd: queue->cmd);
1085	if (unlikely(ret)) {
1086	pr_err("queue %d: failed to map data\n", queue->idx);
1087	if (nvmet_tcp_has_inline_data(cmd: queue->cmd))
1088	nvmet_tcp_fatal_error(queue);
1089	else
1090	nvmet_req_complete(req, status: ret);
1091	ret = -EAGAIN;
1092	goto out;
1093	}
1094
1095	if (nvmet_tcp_need_data_in(cmd: queue->cmd)) {
1096	if (nvmet_tcp_has_inline_data(cmd: queue->cmd)) {
1097	queue->rcv_state = NVMET_TCP_RECV_DATA;
1098	nvmet_tcp_build_pdu_iovec(cmd: queue->cmd);
1099	return 0;
1100	}
1101	/* send back R2T */
1102	nvmet_tcp_queue_response(req: &queue->cmd->req);
1103	goto out;
1104	}
1105
1106	queue->cmd->req.execute(&queue->cmd->req);
1107	out:
1108	nvmet_prepare_receive_pdu(queue);
1109	return ret;
1110	}
1111
1112	static const u8 nvme_tcp_pdu_sizes[] = {
1113	[nvme_tcp_icreq] = sizeof(struct nvme_tcp_icreq_pdu),
1114	[nvme_tcp_cmd] = sizeof(struct nvme_tcp_cmd_pdu),
1115	[nvme_tcp_h2c_data] = sizeof(struct nvme_tcp_data_pdu),
1116	};
1117
1118	static inline u8 nvmet_tcp_pdu_size(u8 type)
1119	{
1120	size_t idx = type;
1121
1122	return (idx < ARRAY_SIZE(nvme_tcp_pdu_sizes) &&
1123	nvme_tcp_pdu_sizes[idx]) ?
1124	nvme_tcp_pdu_sizes[idx] : 0;
1125	}
1126
1127	static inline bool nvmet_tcp_pdu_valid(u8 type)
1128	{
1129	switch (type) {
1130	case nvme_tcp_icreq:
1131	case nvme_tcp_cmd:
1132	case nvme_tcp_h2c_data:
1133	/* fallthru */
1134	return true;
1135	}
1136
1137	return false;
1138	}
1139
1140	static int nvmet_tcp_tls_record_ok(struct nvmet_tcp_queue *queue,
1141	struct msghdr *msg, char *cbuf)
1142	{
1143	struct cmsghdr *cmsg = (struct cmsghdr *)cbuf;
1144	u8 ctype, level, description;
1145	int ret = 0;
1146
1147	ctype = tls_get_record_type(sk: queue->sock->sk, msg: cmsg);
1148	switch (ctype) {
1149	case 0:
1150	break;
1151	case TLS_RECORD_TYPE_DATA:
1152	break;
1153	case TLS_RECORD_TYPE_ALERT:
1154	tls_alert_recv(sk: queue->sock->sk, msg, level: &level, description: &description);
1155	if (level == TLS_ALERT_LEVEL_FATAL) {
1156	pr_err("queue %d: TLS Alert desc %u\n",
1157	queue->idx, description);
1158	ret = -ENOTCONN;
1159	} else {
1160	pr_warn("queue %d: TLS Alert desc %u\n",
1161	queue->idx, description);
1162	ret = -EAGAIN;
1163	}
1164	break;
1165	default:
1166	/* discard this record type */
1167	pr_err("queue %d: TLS record %d unhandled\n",
1168	queue->idx, ctype);
1169	ret = -EAGAIN;
1170	break;
1171	}
1172	return ret;
1173	}
1174
1175	static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue)
1176	{
1177	struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;
1178	int len, ret;
1179	struct kvec iov;
1180	char cbuf[CMSG_LEN(sizeof(char))] = {};
1181	struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1182
1183	recv:
1184	iov.iov_base = (void *)&queue->pdu + queue->offset;
1185	iov.iov_len = queue->left;
1186	if (queue->tls_pskid) {
1187	msg.msg_control = cbuf;
1188	msg.msg_controllen = sizeof(cbuf);
1189	}
1190	len = kernel_recvmsg(sock: queue->sock, msg: &msg, vec: &iov, num: 1,
1191	len: iov.iov_len, flags: msg.msg_flags);
1192	if (unlikely(len < 0))
1193	return len;
1194	if (queue->tls_pskid) {
1195	ret = nvmet_tcp_tls_record_ok(queue, msg: &msg, cbuf);
1196	if (ret < 0)
1197	return ret;
1198	}
1199
1200	queue->offset += len;
1201	queue->left -= len;
1202	if (queue->left)
1203	return -EAGAIN;
1204
1205	if (queue->offset == sizeof(struct nvme_tcp_hdr)) {
1206	u8 hdgst = nvmet_tcp_hdgst_len(queue);
1207
1208	if (unlikely(!nvmet_tcp_pdu_valid(hdr->type))) {
1209	pr_err("unexpected pdu type %d\n", hdr->type);
1210	nvmet_tcp_fatal_error(queue);
1211	return -EIO;
1212	}
1213
1214	if (unlikely(hdr->hlen != nvmet_tcp_pdu_size(hdr->type))) {
1215	pr_err("pdu %d bad hlen %d\n", hdr->type, hdr->hlen);
1216	return -EIO;
1217	}
1218
1219	queue->left = hdr->hlen - queue->offset + hdgst;
1220	goto recv;
1221	}
1222
1223	if (queue->hdr_digest &&
1224	nvmet_tcp_verify_hdgst(queue, pdu: &queue->pdu, len: hdr->hlen)) {
1225	nvmet_tcp_fatal_error(queue); /* fatal */
1226	return -EPROTO;
1227	}
1228
1229	if (queue->data_digest &&
1230	nvmet_tcp_check_ddgst(queue, pdu: &queue->pdu)) {
1231	nvmet_tcp_fatal_error(queue); /* fatal */
1232	return -EPROTO;
1233	}
1234
1235	return nvmet_tcp_done_recv_pdu(queue);
1236	}
1237
1238	static void nvmet_tcp_prep_recv_ddgst(struct nvmet_tcp_cmd *cmd)
1239	{
1240	struct nvmet_tcp_queue *queue = cmd->queue;
1241
1242	nvmet_tcp_calc_ddgst(hash: queue->rcv_hash, cmd);
1243	queue->offset = 0;
1244	queue->left = NVME_TCP_DIGEST_LENGTH;
1245	queue->rcv_state = NVMET_TCP_RECV_DDGST;
1246	}
1247
1248	static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue)
1249	{
1250	struct nvmet_tcp_cmd *cmd = queue->cmd;
1251	int len, ret;
1252
1253	while (msg_data_left(msg: &cmd->recv_msg)) {
1254	len = sock_recvmsg(sock: cmd->queue->sock, msg: &cmd->recv_msg,
1255	flags: cmd->recv_msg.msg_flags);
1256	if (len <= 0)
1257	return len;
1258	if (queue->tls_pskid) {
1259	ret = nvmet_tcp_tls_record_ok(queue: cmd->queue,
1260	msg: &cmd->recv_msg, cbuf: cmd->recv_cbuf);
1261	if (ret < 0)
1262	return ret;
1263	}
1264
1265	cmd->pdu_recv += len;
1266	cmd->rbytes_done += len;
1267	}
1268
1269	if (queue->data_digest) {
1270	nvmet_tcp_prep_recv_ddgst(cmd);
1271	return 0;
1272	}
1273
1274	if (cmd->rbytes_done == cmd->req.transfer_len)
1275	nvmet_tcp_execute_request(cmd);
1276
1277	nvmet_prepare_receive_pdu(queue);
1278	return 0;
1279	}
1280
1281	static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue)
1282	{
1283	struct nvmet_tcp_cmd *cmd = queue->cmd;
1284	int ret, len;
1285	char cbuf[CMSG_LEN(sizeof(char))] = {};
1286	struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1287	struct kvec iov = {
1288	.iov_base = (void *)&cmd->recv_ddgst + queue->offset,
1289	.iov_len = queue->left
1290	};
1291
1292	if (queue->tls_pskid) {
1293	msg.msg_control = cbuf;
1294	msg.msg_controllen = sizeof(cbuf);
1295	}
1296	len = kernel_recvmsg(sock: queue->sock, msg: &msg, vec: &iov, num: 1,
1297	len: iov.iov_len, flags: msg.msg_flags);
1298	if (unlikely(len < 0))
1299	return len;
1300	if (queue->tls_pskid) {
1301	ret = nvmet_tcp_tls_record_ok(queue, msg: &msg, cbuf);
1302	if (ret < 0)
1303	return ret;
1304	}
1305
1306	queue->offset += len;
1307	queue->left -= len;
1308	if (queue->left)
1309	return -EAGAIN;
1310
1311	if (queue->data_digest && cmd->exp_ddgst != cmd->recv_ddgst) {
1312	pr_err("queue %d: cmd %d pdu (%d) data digest error: recv %#x expected %#x\n",
1313	queue->idx, cmd->req.cmd->common.command_id,
1314	queue->pdu.cmd.hdr.type, le32_to_cpu(cmd->recv_ddgst),
1315	le32_to_cpu(cmd->exp_ddgst));
1316	nvmet_req_uninit(req: &cmd->req);
1317	nvmet_tcp_free_cmd_buffers(cmd);
1318	nvmet_tcp_fatal_error(queue);
1319	ret = -EPROTO;
1320	goto out;
1321	}
1322
1323	if (cmd->rbytes_done == cmd->req.transfer_len)
1324	nvmet_tcp_execute_request(cmd);
1325
1326	ret = 0;
1327	out:
1328	nvmet_prepare_receive_pdu(queue);
1329	return ret;
1330	}
1331
1332	static int nvmet_tcp_try_recv_one(struct nvmet_tcp_queue *queue)
1333	{
1334	int result = 0;
1335
1336	if (unlikely(queue->rcv_state == NVMET_TCP_RECV_ERR))
1337	return 0;
1338
1339	if (queue->rcv_state == NVMET_TCP_RECV_PDU) {
1340	result = nvmet_tcp_try_recv_pdu(queue);
1341	if (result != 0)
1342	goto done_recv;
1343	}
1344
1345	if (queue->rcv_state == NVMET_TCP_RECV_DATA) {
1346	result = nvmet_tcp_try_recv_data(queue);
1347	if (result != 0)
1348	goto done_recv;
1349	}
1350
1351	if (queue->rcv_state == NVMET_TCP_RECV_DDGST) {
1352	result = nvmet_tcp_try_recv_ddgst(queue);
1353	if (result != 0)
1354	goto done_recv;
1355	}
1356
1357	done_recv:
1358	if (result < 0) {
1359	if (result == -EAGAIN)
1360	return 0;
1361	return result;
1362	}
1363	return 1;
1364	}
1365
1366	static int nvmet_tcp_try_recv(struct nvmet_tcp_queue *queue,
1367	int budget, int *recvs)
1368	{
1369	int i, ret = 0;
1370
1371	for (i = 0; i < budget; i++) {
1372	ret = nvmet_tcp_try_recv_one(queue);
1373	if (unlikely(ret < 0)) {
1374	nvmet_tcp_socket_error(queue, status: ret);
1375	goto done;
1376	} else if (ret == 0) {
1377	break;
1378	}
1379	(*recvs)++;
1380	}
1381	done:
1382	return ret;
1383	}
1384
1385	static void nvmet_tcp_release_queue(struct kref *kref)
1386	{
1387	struct nvmet_tcp_queue *queue =
1388	container_of(kref, struct nvmet_tcp_queue, kref);
1389
1390	WARN_ON(queue->state != NVMET_TCP_Q_DISCONNECTING);
1391	queue_work(wq: nvmet_wq, work: &queue->release_work);
1392	}
1393
1394	static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue)
1395	{
1396	spin_lock_bh(lock: &queue->state_lock);
1397	if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) {
1398	/* Socket closed during handshake */
1399	tls_handshake_cancel(sk: queue->sock->sk);
1400	}
1401	if (queue->state != NVMET_TCP_Q_DISCONNECTING) {
1402	queue->state = NVMET_TCP_Q_DISCONNECTING;
1403	kref_put(kref: &queue->kref, release: nvmet_tcp_release_queue);
1404	}
1405	spin_unlock_bh(lock: &queue->state_lock);
1406	}
1407
1408	static inline void nvmet_tcp_arm_queue_deadline(struct nvmet_tcp_queue *queue)
1409	{
1410	queue->poll_end = jiffies + usecs_to_jiffies(u: idle_poll_period_usecs);
1411	}
1412
1413	static bool nvmet_tcp_check_queue_deadline(struct nvmet_tcp_queue *queue,
1414	int ops)
1415	{
1416	if (!idle_poll_period_usecs)
1417	return false;
1418
1419	if (ops)
1420	nvmet_tcp_arm_queue_deadline(queue);
1421
1422	return !time_after(jiffies, queue->poll_end);
1423	}
1424
1425	static void nvmet_tcp_io_work(struct work_struct *w)
1426	{
1427	struct nvmet_tcp_queue *queue =
1428	container_of(w, struct nvmet_tcp_queue, io_work);
1429	bool pending;
1430	int ret, ops = 0;
1431
1432	do {
1433	pending = false;
1434
1435	ret = nvmet_tcp_try_recv(queue, NVMET_TCP_RECV_BUDGET, recvs: &ops);
1436	if (ret > 0)
1437	pending = true;
1438	else if (ret < 0)
1439	return;
1440
1441	ret = nvmet_tcp_try_send(queue, NVMET_TCP_SEND_BUDGET, sends: &ops);
1442	if (ret > 0)
1443	pending = true;
1444	else if (ret < 0)
1445	return;
1446
1447	} while (pending && ops < NVMET_TCP_IO_WORK_BUDGET);
1448
1449	/*
1450	* Requeue the worker if idle deadline period is in progress or any
1451	* ops activity was recorded during the do-while loop above.
1452	*/
1453	if (nvmet_tcp_check_queue_deadline(queue, ops) || pending)
1454	queue_work_on(cpu: queue_cpu(queue), wq: nvmet_tcp_wq, work: &queue->io_work);
1455	}
1456
1457	static int nvmet_tcp_alloc_cmd(struct nvmet_tcp_queue *queue,
1458	struct nvmet_tcp_cmd *c)
1459	{
1460	u8 hdgst = nvmet_tcp_hdgst_len(queue);
1461
1462	c->queue = queue;
1463	c->req.port = queue->port->nport;
1464
1465	c->cmd_pdu = page_frag_alloc(nc: &queue->pf_cache,
1466	fragsz: sizeof(*c->cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
1467	if (!c->cmd_pdu)
1468	return -ENOMEM;
1469	c->req.cmd = &c->cmd_pdu->cmd;
1470
1471	c->rsp_pdu = page_frag_alloc(nc: &queue->pf_cache,
1472	fragsz: sizeof(*c->rsp_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
1473	if (!c->rsp_pdu)
1474	goto out_free_cmd;
1475	c->req.cqe = &c->rsp_pdu->cqe;
1476
1477	c->data_pdu = page_frag_alloc(nc: &queue->pf_cache,
1478	fragsz: sizeof(*c->data_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
1479	if (!c->data_pdu)
1480	goto out_free_rsp;
1481
1482	c->r2t_pdu = page_frag_alloc(nc: &queue->pf_cache,
1483	fragsz: sizeof(*c->r2t_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
1484	if (!c->r2t_pdu)
1485	goto out_free_data;
1486
1487	if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) {
1488	c->recv_msg.msg_control = c->recv_cbuf;
1489	c->recv_msg.msg_controllen = sizeof(c->recv_cbuf);
1490	}
1491	c->recv_msg.msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL;
1492
1493	list_add_tail(new: &c->entry, head: &queue->free_list);
1494
1495	return 0;
1496	out_free_data:
1497	page_frag_free(addr: c->data_pdu);
1498	out_free_rsp:
1499	page_frag_free(addr: c->rsp_pdu);
1500	out_free_cmd:
1501	page_frag_free(addr: c->cmd_pdu);
1502	return -ENOMEM;
1503	}
1504
1505	static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c)
1506	{
1507	page_frag_free(addr: c->r2t_pdu);
1508	page_frag_free(addr: c->data_pdu);
1509	page_frag_free(addr: c->rsp_pdu);
1510	page_frag_free(addr: c->cmd_pdu);
1511	}
1512
1513	static int nvmet_tcp_alloc_cmds(struct nvmet_tcp_queue *queue)
1514	{
1515	struct nvmet_tcp_cmd *cmds;
1516	int i, ret = -EINVAL, nr_cmds = queue->nr_cmds;
1517
1518	cmds = kcalloc(n: nr_cmds, size: sizeof(struct nvmet_tcp_cmd), GFP_KERNEL);
1519	if (!cmds)
1520	goto out;
1521
1522	for (i = 0; i < nr_cmds; i++) {
1523	ret = nvmet_tcp_alloc_cmd(queue, c: cmds + i);
1524	if (ret)
1525	goto out_free;
1526	}
1527
1528	queue->cmds = cmds;
1529
1530	return 0;
1531	out_free:
1532	while (--i >= 0)
1533	nvmet_tcp_free_cmd(c: cmds + i);
1534	kfree(objp: cmds);
1535	out:
1536	return ret;
1537	}
1538
1539	static void nvmet_tcp_free_cmds(struct nvmet_tcp_queue *queue)
1540	{
1541	struct nvmet_tcp_cmd *cmds = queue->cmds;
1542	int i;
1543
1544	for (i = 0; i < queue->nr_cmds; i++)
1545	nvmet_tcp_free_cmd(c: cmds + i);
1546
1547	nvmet_tcp_free_cmd(c: &queue->connect);
1548	kfree(objp: cmds);
1549	}
1550
1551	static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue)
1552	{
1553	struct socket *sock = queue->sock;
1554
1555	write_lock_bh(&sock->sk->sk_callback_lock);
1556	sock->sk->sk_data_ready = queue->data_ready;
1557	sock->sk->sk_state_change = queue->state_change;
1558	sock->sk->sk_write_space = queue->write_space;
1559	sock->sk->sk_user_data = NULL;
1560	write_unlock_bh(&sock->sk->sk_callback_lock);
1561	}
1562
1563	static void nvmet_tcp_uninit_data_in_cmds(struct nvmet_tcp_queue *queue)
1564	{
1565	struct nvmet_tcp_cmd *cmd = queue->cmds;
1566	int i;
1567
1568	for (i = 0; i < queue->nr_cmds; i++, cmd++) {
1569	if (nvmet_tcp_need_data_in(cmd))
1570	nvmet_req_uninit(req: &cmd->req);
1571	}
1572
1573	if (!queue->nr_cmds && nvmet_tcp_need_data_in(cmd: &queue->connect)) {
1574	/* failed in connect */
1575	nvmet_req_uninit(req: &queue->connect.req);
1576	}
1577	}
1578
1579	static void nvmet_tcp_free_cmd_data_in_buffers(struct nvmet_tcp_queue *queue)
1580	{
1581	struct nvmet_tcp_cmd *cmd = queue->cmds;
1582	int i;
1583
1584	for (i = 0; i < queue->nr_cmds; i++, cmd++) {
1585	if (nvmet_tcp_need_data_in(cmd))
1586	nvmet_tcp_free_cmd_buffers(cmd);
1587	}
1588
1589	if (!queue->nr_cmds && nvmet_tcp_need_data_in(cmd: &queue->connect))
1590	nvmet_tcp_free_cmd_buffers(cmd: &queue->connect);
1591	}
1592
1593	static void nvmet_tcp_release_queue_work(struct work_struct *w)
1594	{
1595	struct nvmet_tcp_queue *queue =
1596	container_of(w, struct nvmet_tcp_queue, release_work);
1597
1598	mutex_lock(&nvmet_tcp_queue_mutex);
1599	list_del_init(entry: &queue->queue_list);
1600	mutex_unlock(lock: &nvmet_tcp_queue_mutex);
1601
1602	nvmet_tcp_restore_socket_callbacks(queue);
1603	cancel_delayed_work_sync(dwork: &queue->tls_handshake_tmo_work);
1604	cancel_work_sync(work: &queue->io_work);
1605	/* stop accepting incoming data */
1606	queue->rcv_state = NVMET_TCP_RECV_ERR;
1607
1608	nvmet_tcp_uninit_data_in_cmds(queue);
1609	nvmet_sq_destroy(sq: &queue->nvme_sq);
1610	cancel_work_sync(work: &queue->io_work);
1611	nvmet_tcp_free_cmd_data_in_buffers(queue);
1612	/* ->sock will be released by fput() */
1613	fput(queue->sock->file);
1614	nvmet_tcp_free_cmds(queue);
1615	if (queue->hdr_digest || queue->data_digest)
1616	nvmet_tcp_free_crypto(queue);
1617	ida_free(&nvmet_tcp_queue_ida, id: queue->idx);
1618	page_frag_cache_drain(nc: &queue->pf_cache);
1619	kfree(objp: queue);
1620	}
1621
1622	static void nvmet_tcp_data_ready(struct sock *sk)
1623	{
1624	struct nvmet_tcp_queue *queue;
1625
1626	trace_sk_data_ready(sk);
1627
1628	read_lock_bh(&sk->sk_callback_lock);
1629	queue = sk->sk_user_data;
1630	if (likely(queue)) {
1631	if (queue->data_ready)
1632	queue->data_ready(sk);
1633	if (queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)
1634	queue_work_on(cpu: queue_cpu(queue), wq: nvmet_tcp_wq,
1635	work: &queue->io_work);
1636	}
1637	read_unlock_bh(&sk->sk_callback_lock);
1638	}
1639
1640	static void nvmet_tcp_write_space(struct sock *sk)
1641	{
1642	struct nvmet_tcp_queue *queue;
1643
1644	read_lock_bh(&sk->sk_callback_lock);
1645	queue = sk->sk_user_data;
1646	if (unlikely(!queue))
1647	goto out;
1648
1649	if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) {
1650	queue->write_space(sk);
1651	goto out;
1652	}
1653
1654	if (sk_stream_is_writeable(sk)) {
1655	clear_bit(SOCK_NOSPACE, addr: &sk->sk_socket->flags);
1656	queue_work_on(cpu: queue_cpu(queue), wq: nvmet_tcp_wq, work: &queue->io_work);
1657	}
1658	out:
1659	read_unlock_bh(&sk->sk_callback_lock);
1660	}
1661
1662	static void nvmet_tcp_state_change(struct sock *sk)
1663	{
1664	struct nvmet_tcp_queue *queue;
1665
1666	read_lock_bh(&sk->sk_callback_lock);
1667	queue = sk->sk_user_data;
1668	if (!queue)
1669	goto done;
1670
1671	switch (sk->sk_state) {
1672	case TCP_FIN_WAIT2:
1673	case TCP_LAST_ACK:
1674	break;
1675	case TCP_FIN_WAIT1:
1676	case TCP_CLOSE_WAIT:
1677	case TCP_CLOSE:
1678	/* FALLTHRU */
1679	nvmet_tcp_schedule_release_queue(queue);
1680	break;
1681	default:
1682	pr_warn("queue %d unhandled state %d\n",
1683	queue->idx, sk->sk_state);
1684	}
1685	done:
1686	read_unlock_bh(&sk->sk_callback_lock);
1687	}
1688
1689	static int nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue *queue)
1690	{
1691	struct socket *sock = queue->sock;
1692	struct inet_sock *inet = inet_sk(sock->sk);
1693	int ret;
1694
1695	ret = kernel_getsockname(sock,
1696	addr: (struct sockaddr *)&queue->sockaddr);
1697	if (ret < 0)
1698	return ret;
1699
1700	ret = kernel_getpeername(sock,
1701	addr: (struct sockaddr *)&queue->sockaddr_peer);
1702	if (ret < 0)
1703	return ret;
1704
1705	/*
1706	* Cleanup whatever is sitting in the TCP transmit queue on socket
1707	* close. This is done to prevent stale data from being sent should
1708	* the network connection be restored before TCP times out.
1709	*/
1710	sock_no_linger(sk: sock->sk);
1711
1712	if (so_priority > 0)
1713	sock_set_priority(sk: sock->sk, priority: so_priority);
1714
1715	/* Set socket type of service */
1716	if (inet->rcv_tos > 0)
1717	ip_sock_set_tos(sk: sock->sk, val: inet->rcv_tos);
1718
1719	ret = 0;
1720	write_lock_bh(&sock->sk->sk_callback_lock);
1721	if (sock->sk->sk_state != TCP_ESTABLISHED) {
1722	/*
1723	* If the socket is already closing, don't even start
1724	* consuming it
1725	*/
1726	ret = -ENOTCONN;
1727	} else {
1728	sock->sk->sk_user_data = queue;
1729	queue->data_ready = sock->sk->sk_data_ready;
1730	sock->sk->sk_data_ready = nvmet_tcp_data_ready;
1731	queue->state_change = sock->sk->sk_state_change;
1732	sock->sk->sk_state_change = nvmet_tcp_state_change;
1733	queue->write_space = sock->sk->sk_write_space;
1734	sock->sk->sk_write_space = nvmet_tcp_write_space;
1735	if (idle_poll_period_usecs)
1736	nvmet_tcp_arm_queue_deadline(queue);
1737	queue_work_on(cpu: queue_cpu(queue), wq: nvmet_tcp_wq, work: &queue->io_work);
1738	}
1739	write_unlock_bh(&sock->sk->sk_callback_lock);
1740
1741	return ret;
1742	}
1743
1744	#ifdef CONFIG_NVME_TARGET_TCP_TLS
1745	static int nvmet_tcp_try_peek_pdu(struct nvmet_tcp_queue *queue)
1746	{
1747	struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;
1748	int len, ret;
1749	struct kvec iov = {
1750	.iov_base = (u8 *)&queue->pdu + queue->offset,
1751	.iov_len = sizeof(struct nvme_tcp_hdr),
1752	};
1753	char cbuf[CMSG_LEN(sizeof(char))] = {};
1754	struct msghdr msg = {
1755	.msg_control = cbuf,
1756	.msg_controllen = sizeof(cbuf),
1757	.msg_flags = MSG_PEEK,
1758	};
1759
1760	if (nvmet_port_secure_channel_required(port: queue->port->nport))
1761	return 0;
1762
1763	len = kernel_recvmsg(sock: queue->sock, msg: &msg, vec: &iov, num: 1,
1764	len: iov.iov_len, flags: msg.msg_flags);
1765	if (unlikely(len < 0)) {
1766	pr_debug("queue %d: peek error %d\n",
1767	queue->idx, len);
1768	return len;
1769	}
1770
1771	ret = nvmet_tcp_tls_record_ok(queue, msg: &msg, cbuf);
1772	if (ret < 0)
1773	return ret;
1774
1775	if (len < sizeof(struct nvme_tcp_hdr)) {
1776	pr_debug("queue %d: short read, %d bytes missing\n",
1777	queue->idx, (int)iov.iov_len - len);
1778	return -EAGAIN;
1779	}
1780	pr_debug("queue %d: hdr type %d hlen %d plen %d size %d\n",
1781	queue->idx, hdr->type, hdr->hlen, hdr->plen,
1782	(int)sizeof(struct nvme_tcp_icreq_pdu));
1783	if (hdr->type == nvme_tcp_icreq &&
1784	hdr->hlen == sizeof(struct nvme_tcp_icreq_pdu) &&
1785	hdr->plen == cpu_to_le32(sizeof(struct nvme_tcp_icreq_pdu))) {
1786	pr_debug("queue %d: icreq detected\n",
1787	queue->idx);
1788	return len;
1789	}
1790	return 0;
1791	}
1792
1793	static void nvmet_tcp_tls_handshake_done(void *data, int status,
1794	key_serial_t peerid)
1795	{
1796	struct nvmet_tcp_queue *queue = data;
1797
1798	pr_debug("queue %d: TLS handshake done, key %x, status %d\n",
1799	queue->idx, peerid, status);
1800	spin_lock_bh(lock: &queue->state_lock);
1801	if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) {
1802	spin_unlock_bh(lock: &queue->state_lock);
1803	return;
1804	}
1805	if (!status) {
1806	queue->tls_pskid = peerid;
1807	queue->state = NVMET_TCP_Q_CONNECTING;
1808	} else
1809	queue->state = NVMET_TCP_Q_FAILED;
1810	spin_unlock_bh(lock: &queue->state_lock);
1811
1812	cancel_delayed_work_sync(dwork: &queue->tls_handshake_tmo_work);
1813	if (status)
1814	nvmet_tcp_schedule_release_queue(queue);
1815	else
1816	nvmet_tcp_set_queue_sock(queue);
1817	kref_put(kref: &queue->kref, release: nvmet_tcp_release_queue);
1818	}
1819
1820	static void nvmet_tcp_tls_handshake_timeout(struct work_struct *w)
1821	{
1822	struct nvmet_tcp_queue *queue = container_of(to_delayed_work(w),
1823	struct nvmet_tcp_queue, tls_handshake_tmo_work);
1824
1825	pr_warn("queue %d: TLS handshake timeout\n", queue->idx);
1826	/*
1827	* If tls_handshake_cancel() fails we've lost the race with
1828	* nvmet_tcp_tls_handshake_done() */
1829	if (!tls_handshake_cancel(sk: queue->sock->sk))
1830	return;
1831	spin_lock_bh(lock: &queue->state_lock);
1832	if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) {
1833	spin_unlock_bh(lock: &queue->state_lock);
1834	return;
1835	}
1836	queue->state = NVMET_TCP_Q_FAILED;
1837	spin_unlock_bh(lock: &queue->state_lock);
1838	nvmet_tcp_schedule_release_queue(queue);
1839	kref_put(kref: &queue->kref, release: nvmet_tcp_release_queue);
1840	}
1841
1842	static int nvmet_tcp_tls_handshake(struct nvmet_tcp_queue *queue)
1843	{
1844	int ret = -EOPNOTSUPP;
1845	struct tls_handshake_args args;
1846
1847	if (queue->state != NVMET_TCP_Q_TLS_HANDSHAKE) {
1848	pr_warn("cannot start TLS in state %d\n", queue->state);
1849	return -EINVAL;
1850	}
1851
1852	kref_get(kref: &queue->kref);
1853	pr_debug("queue %d: TLS ServerHello\n", queue->idx);
1854	memset(&args, 0, sizeof(args));
1855	args.ta_sock = queue->sock;
1856	args.ta_done = nvmet_tcp_tls_handshake_done;
1857	args.ta_data = queue;
1858	args.ta_keyring = key_serial(key: queue->port->nport->keyring);
1859	args.ta_timeout_ms = tls_handshake_timeout * 1000;
1860
1861	ret = tls_server_hello_psk(args: &args, GFP_KERNEL);
1862	if (ret) {
1863	kref_put(kref: &queue->kref, release: nvmet_tcp_release_queue);
1864	pr_err("failed to start TLS, err=%d\n", ret);
1865	} else {
1866	queue_delayed_work(wq: nvmet_wq, dwork: &queue->tls_handshake_tmo_work,
1867	delay: tls_handshake_timeout * HZ);
1868	}
1869	return ret;
1870	}
1871	#else
1872	static void nvmet_tcp_tls_handshake_timeout(struct work_struct *w) {}
1873	#endif
1874
1875	static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port,
1876	struct socket *newsock)
1877	{
1878	struct nvmet_tcp_queue *queue;
1879	struct file *sock_file = NULL;
1880	int ret;
1881
1882	queue = kzalloc(size: sizeof(*queue), GFP_KERNEL);
1883	if (!queue) {
1884	ret = -ENOMEM;
1885	goto out_release;
1886	}
1887
1888	INIT_WORK(&queue->release_work, nvmet_tcp_release_queue_work);
1889	INIT_WORK(&queue->io_work, nvmet_tcp_io_work);
1890	kref_init(kref: &queue->kref);
1891	queue->sock = newsock;
1892	queue->port = port;
1893	queue->nr_cmds = 0;
1894	spin_lock_init(&queue->state_lock);
1895	if (queue->port->nport->disc_addr.tsas.tcp.sectype ==
1896	NVMF_TCP_SECTYPE_TLS13)
1897	queue->state = NVMET_TCP_Q_TLS_HANDSHAKE;
1898	else
1899	queue->state = NVMET_TCP_Q_CONNECTING;
1900	INIT_LIST_HEAD(list: &queue->free_list);
1901	init_llist_head(list: &queue->resp_list);
1902	INIT_LIST_HEAD(list: &queue->resp_send_list);
1903
1904	sock_file = sock_alloc_file(sock: queue->sock, O_CLOEXEC, NULL);
1905	if (IS_ERR(ptr: sock_file)) {
1906	ret = PTR_ERR(ptr: sock_file);
1907	goto out_free_queue;
1908	}
1909
1910	queue->idx = ida_alloc(ida: &nvmet_tcp_queue_ida, GFP_KERNEL);
1911	if (queue->idx < 0) {
1912	ret = queue->idx;
1913	goto out_sock;
1914	}
1915
1916	ret = nvmet_tcp_alloc_cmd(queue, c: &queue->connect);
1917	if (ret)
1918	goto out_ida_remove;
1919
1920	ret = nvmet_sq_init(sq: &queue->nvme_sq);
1921	if (ret)
1922	goto out_free_connect;
1923
1924	nvmet_prepare_receive_pdu(queue);
1925
1926	mutex_lock(&nvmet_tcp_queue_mutex);
1927	list_add_tail(new: &queue->queue_list, head: &nvmet_tcp_queue_list);
1928	mutex_unlock(lock: &nvmet_tcp_queue_mutex);
1929
1930	INIT_DELAYED_WORK(&queue->tls_handshake_tmo_work,
1931	nvmet_tcp_tls_handshake_timeout);
1932	#ifdef CONFIG_NVME_TARGET_TCP_TLS
1933	if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) {
1934	struct sock *sk = queue->sock->sk;
1935
1936	/* Restore the default callbacks before starting upcall */
1937	read_lock_bh(&sk->sk_callback_lock);
1938	sk->sk_user_data = NULL;
1939	sk->sk_data_ready = port->data_ready;
1940	read_unlock_bh(&sk->sk_callback_lock);
1941	if (!nvmet_tcp_try_peek_pdu(queue)) {
1942	if (!nvmet_tcp_tls_handshake(queue))
1943	return;
1944	/* TLS handshake failed, terminate the connection */
1945	goto out_destroy_sq;
1946	}
1947	/* Not a TLS connection, continue with normal processing */
1948	queue->state = NVMET_TCP_Q_CONNECTING;
1949	}
1950	#endif
1951
1952	ret = nvmet_tcp_set_queue_sock(queue);
1953	if (ret)
1954	goto out_destroy_sq;
1955
1956	return;
1957	out_destroy_sq:
1958	mutex_lock(&nvmet_tcp_queue_mutex);
1959	list_del_init(entry: &queue->queue_list);
1960	mutex_unlock(lock: &nvmet_tcp_queue_mutex);
1961	nvmet_sq_destroy(sq: &queue->nvme_sq);
1962	out_free_connect:
1963	nvmet_tcp_free_cmd(c: &queue->connect);
1964	out_ida_remove:
1965	ida_free(&nvmet_tcp_queue_ida, id: queue->idx);
1966	out_sock:
1967	fput(queue->sock->file);
1968	out_free_queue:
1969	kfree(objp: queue);
1970	out_release:
1971	pr_err("failed to allocate queue, error %d\n", ret);
1972	if (!sock_file)
1973	sock_release(sock: newsock);
1974	}
1975
1976	static void nvmet_tcp_accept_work(struct work_struct *w)
1977	{
1978	struct nvmet_tcp_port *port =
1979	container_of(w, struct nvmet_tcp_port, accept_work);
1980	struct socket *newsock;
1981	int ret;
1982
1983	while (true) {
1984	ret = kernel_accept(sock: port->sock, newsock: &newsock, O_NONBLOCK);
1985	if (ret < 0) {
1986	if (ret != -EAGAIN)
1987	pr_warn("failed to accept err=%d\n", ret);
1988	return;
1989	}
1990	nvmet_tcp_alloc_queue(port, newsock);
1991	}
1992	}
1993
1994	static void nvmet_tcp_listen_data_ready(struct sock *sk)
1995	{
1996	struct nvmet_tcp_port *port;
1997
1998	trace_sk_data_ready(sk);
1999
2000	read_lock_bh(&sk->sk_callback_lock);
2001	port = sk->sk_user_data;
2002	if (!port)
2003	goto out;
2004
2005	if (sk->sk_state == TCP_LISTEN)
2006	queue_work(wq: nvmet_wq, work: &port->accept_work);
2007	out:
2008	read_unlock_bh(&sk->sk_callback_lock);
2009	}
2010
2011	static int nvmet_tcp_add_port(struct nvmet_port *nport)
2012	{
2013	struct nvmet_tcp_port *port;
2014	__kernel_sa_family_t af;
2015	int ret;
2016
2017	port = kzalloc(size: sizeof(*port), GFP_KERNEL);
2018	if (!port)
2019	return -ENOMEM;
2020
2021	switch (nport->disc_addr.adrfam) {
2022	case NVMF_ADDR_FAMILY_IP4:
2023	af = AF_INET;
2024	break;
2025	case NVMF_ADDR_FAMILY_IP6:
2026	af = AF_INET6;
2027	break;
2028	default:
2029	pr_err("address family %d not supported\n",
2030	nport->disc_addr.adrfam);
2031	ret = -EINVAL;
2032	goto err_port;
2033	}
2034
2035	ret = inet_pton_with_scope(net: &init_net, af, src: nport->disc_addr.traddr,
2036	port: nport->disc_addr.trsvcid, addr: &port->addr);
2037	if (ret) {
2038	pr_err("malformed ip/port passed: %s:%s\n",
2039	nport->disc_addr.traddr, nport->disc_addr.trsvcid);
2040	goto err_port;
2041	}
2042
2043	port->nport = nport;
2044	INIT_WORK(&port->accept_work, nvmet_tcp_accept_work);
2045	if (port->nport->inline_data_size < 0)
2046	port->nport->inline_data_size = NVMET_TCP_DEF_INLINE_DATA_SIZE;
2047
2048	ret = sock_create(family: port->addr.ss_family, type: SOCK_STREAM,
2049	IPPROTO_TCP, res: &port->sock);
2050	if (ret) {
2051	pr_err("failed to create a socket\n");
2052	goto err_port;
2053	}
2054
2055	port->sock->sk->sk_user_data = port;
2056	port->data_ready = port->sock->sk->sk_data_ready;
2057	port->sock->sk->sk_data_ready = nvmet_tcp_listen_data_ready;
2058	sock_set_reuseaddr(sk: port->sock->sk);
2059	tcp_sock_set_nodelay(sk: port->sock->sk);
2060	if (so_priority > 0)
2061	sock_set_priority(sk: port->sock->sk, priority: so_priority);
2062
2063	ret = kernel_bind(sock: port->sock, addr: (struct sockaddr *)&port->addr,
2064	addrlen: sizeof(port->addr));
2065	if (ret) {
2066	pr_err("failed to bind port socket %d\n", ret);
2067	goto err_sock;
2068	}
2069
2070	ret = kernel_listen(sock: port->sock, NVMET_TCP_BACKLOG);
2071	if (ret) {
2072	pr_err("failed to listen %d on port sock\n", ret);
2073	goto err_sock;
2074	}
2075
2076	nport->priv = port;
2077	pr_info("enabling port %d (%pISpc)\n",
2078	le16_to_cpu(nport->disc_addr.portid), &port->addr);
2079
2080	return 0;
2081
2082	err_sock:
2083	sock_release(sock: port->sock);
2084	err_port:
2085	kfree(objp: port);
2086	return ret;
2087	}
2088
2089	static void nvmet_tcp_destroy_port_queues(struct nvmet_tcp_port *port)
2090	{
2091	struct nvmet_tcp_queue *queue;
2092
2093	mutex_lock(&nvmet_tcp_queue_mutex);
2094	list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
2095	if (queue->port == port)
2096	kernel_sock_shutdown(sock: queue->sock, how: SHUT_RDWR);
2097	mutex_unlock(lock: &nvmet_tcp_queue_mutex);
2098	}
2099
2100	static void nvmet_tcp_remove_port(struct nvmet_port *nport)
2101	{
2102	struct nvmet_tcp_port *port = nport->priv;
2103
2104	write_lock_bh(&port->sock->sk->sk_callback_lock);
2105	port->sock->sk->sk_data_ready = port->data_ready;
2106	port->sock->sk->sk_user_data = NULL;
2107	write_unlock_bh(&port->sock->sk->sk_callback_lock);
2108	cancel_work_sync(work: &port->accept_work);
2109	/*
2110	* Destroy the remaining queues, which are not belong to any
2111	* controller yet.
2112	*/
2113	nvmet_tcp_destroy_port_queues(port);
2114
2115	sock_release(sock: port->sock);
2116	kfree(objp: port);
2117	}
2118
2119	static void nvmet_tcp_delete_ctrl(struct nvmet_ctrl *ctrl)
2120	{
2121	struct nvmet_tcp_queue *queue;
2122
2123	mutex_lock(&nvmet_tcp_queue_mutex);
2124	list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
2125	if (queue->nvme_sq.ctrl == ctrl)
2126	kernel_sock_shutdown(sock: queue->sock, how: SHUT_RDWR);
2127	mutex_unlock(lock: &nvmet_tcp_queue_mutex);
2128	}
2129
2130	static u16 nvmet_tcp_install_queue(struct nvmet_sq *sq)
2131	{
2132	struct nvmet_tcp_queue *queue =
2133	container_of(sq, struct nvmet_tcp_queue, nvme_sq);
2134
2135	if (sq->qid == 0) {
2136	struct nvmet_tcp_queue *q;
2137	int pending = 0;
2138
2139	/* Check for pending controller teardown */
2140	mutex_lock(&nvmet_tcp_queue_mutex);
2141	list_for_each_entry(q, &nvmet_tcp_queue_list, queue_list) {
2142	if (q->nvme_sq.ctrl == sq->ctrl &&
2143	q->state == NVMET_TCP_Q_DISCONNECTING)
2144	pending++;
2145	}
2146	mutex_unlock(lock: &nvmet_tcp_queue_mutex);
2147	if (pending > NVMET_TCP_BACKLOG)
2148	return NVME_SC_CONNECT_CTRL_BUSY;
2149	}
2150
2151	queue->nr_cmds = sq->size * 2;
2152	if (nvmet_tcp_alloc_cmds(queue))
2153	return NVME_SC_INTERNAL;
2154	return 0;
2155	}
2156
2157	static void nvmet_tcp_disc_port_addr(struct nvmet_req *req,
2158	struct nvmet_port *nport, char *traddr)
2159	{
2160	struct nvmet_tcp_port *port = nport->priv;
2161
2162	if (inet_addr_is_any(addr: (struct sockaddr *)&port->addr)) {
2163	struct nvmet_tcp_cmd *cmd =
2164	container_of(req, struct nvmet_tcp_cmd, req);
2165	struct nvmet_tcp_queue *queue = cmd->queue;
2166
2167	sprintf(buf: traddr, fmt: "%pISc", (struct sockaddr *)&queue->sockaddr);
2168	} else {
2169	memcpy(traddr, nport->disc_addr.traddr, NVMF_TRADDR_SIZE);
2170	}
2171	}
2172
2173	static const struct nvmet_fabrics_ops nvmet_tcp_ops = {
2174	.owner = THIS_MODULE,
2175	.type = NVMF_TRTYPE_TCP,
2176	.msdbd = 1,
2177	.add_port = nvmet_tcp_add_port,
2178	.remove_port = nvmet_tcp_remove_port,
2179	.queue_response = nvmet_tcp_queue_response,
2180	.delete_ctrl = nvmet_tcp_delete_ctrl,
2181	.install_queue = nvmet_tcp_install_queue,
2182	.disc_traddr = nvmet_tcp_disc_port_addr,
2183	};
2184
2185	static int __init nvmet_tcp_init(void)
2186	{
2187	int ret;
2188
2189	nvmet_tcp_wq = alloc_workqueue(fmt: "nvmet_tcp_wq",
2190	flags: WQ_MEM_RECLAIM | WQ_HIGHPRI, max_active: 0);
2191	if (!nvmet_tcp_wq)
2192	return -ENOMEM;
2193
2194	ret = nvmet_register_transport(ops: &nvmet_tcp_ops);
2195	if (ret)
2196	goto err;
2197
2198	return 0;
2199	err:
2200	destroy_workqueue(wq: nvmet_tcp_wq);
2201	return ret;
2202	}
2203
2204	static void __exit nvmet_tcp_exit(void)
2205	{
2206	struct nvmet_tcp_queue *queue;
2207
2208	nvmet_unregister_transport(ops: &nvmet_tcp_ops);
2209
2210	flush_workqueue(nvmet_wq);
2211	mutex_lock(&nvmet_tcp_queue_mutex);
2212	list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
2213	kernel_sock_shutdown(sock: queue->sock, how: SHUT_RDWR);
2214	mutex_unlock(lock: &nvmet_tcp_queue_mutex);
2215	flush_workqueue(nvmet_wq);
2216
2217	destroy_workqueue(wq: nvmet_tcp_wq);
2218	ida_destroy(ida: &nvmet_tcp_queue_ida);
2219	}
2220
2221	module_init(nvmet_tcp_init);
2222	module_exit(nvmet_tcp_exit);
2223
2224	MODULE_DESCRIPTION("NVMe target TCP transport driver");
2225	MODULE_LICENSE("GPL v2");
2226	MODULE_ALIAS("nvmet-transport-3"); /* 3 == NVMF_TRTYPE_TCP */
2227