1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Ceph msgr2 protocol implementation
4 *
5 * Copyright (C) 2020 Ilya Dryomov <idryomov@gmail.com>
6 */
7
8#include <linux/ceph/ceph_debug.h>
9
10#include <crypto/aead.h>
11#include <crypto/hash.h>
12#include <crypto/sha2.h>
13#include <crypto/utils.h>
14#include <linux/bvec.h>
15#include <linux/crc32c.h>
16#include <linux/net.h>
17#include <linux/scatterlist.h>
18#include <linux/socket.h>
19#include <linux/sched/mm.h>
20#include <net/sock.h>
21#include <net/tcp.h>
22
23#include <linux/ceph/ceph_features.h>
24#include <linux/ceph/decode.h>
25#include <linux/ceph/libceph.h>
26#include <linux/ceph/messenger.h>
27
28#include "crypto.h" /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */
29
30#define FRAME_TAG_HELLO 1
31#define FRAME_TAG_AUTH_REQUEST 2
32#define FRAME_TAG_AUTH_BAD_METHOD 3
33#define FRAME_TAG_AUTH_REPLY_MORE 4
34#define FRAME_TAG_AUTH_REQUEST_MORE 5
35#define FRAME_TAG_AUTH_DONE 6
36#define FRAME_TAG_AUTH_SIGNATURE 7
37#define FRAME_TAG_CLIENT_IDENT 8
38#define FRAME_TAG_SERVER_IDENT 9
39#define FRAME_TAG_IDENT_MISSING_FEATURES 10
40#define FRAME_TAG_SESSION_RECONNECT 11
41#define FRAME_TAG_SESSION_RESET 12
42#define FRAME_TAG_SESSION_RETRY 13
43#define FRAME_TAG_SESSION_RETRY_GLOBAL 14
44#define FRAME_TAG_SESSION_RECONNECT_OK 15
45#define FRAME_TAG_WAIT 16
46#define FRAME_TAG_MESSAGE 17
47#define FRAME_TAG_KEEPALIVE2 18
48#define FRAME_TAG_KEEPALIVE2_ACK 19
49#define FRAME_TAG_ACK 20
50
51#define FRAME_LATE_STATUS_ABORTED 0x1
52#define FRAME_LATE_STATUS_COMPLETE 0xe
53#define FRAME_LATE_STATUS_ABORTED_MASK 0xf
54
55#define IN_S_HANDLE_PREAMBLE 1
56#define IN_S_HANDLE_CONTROL 2
57#define IN_S_HANDLE_CONTROL_REMAINDER 3
58#define IN_S_PREPARE_READ_DATA 4
59#define IN_S_PREPARE_READ_DATA_CONT 5
60#define IN_S_PREPARE_READ_ENC_PAGE 6
61#define IN_S_PREPARE_SPARSE_DATA 7
62#define IN_S_PREPARE_SPARSE_DATA_CONT 8
63#define IN_S_HANDLE_EPILOGUE 9
64#define IN_S_FINISH_SKIP 10
65
66#define OUT_S_QUEUE_DATA 1
67#define OUT_S_QUEUE_DATA_CONT 2
68#define OUT_S_QUEUE_ENC_PAGE 3
69#define OUT_S_QUEUE_ZEROS 4
70#define OUT_S_FINISH_MESSAGE 5
71#define OUT_S_GET_NEXT 6
72
73#define CTRL_BODY(p) ((void *)(p) + CEPH_PREAMBLE_LEN)
74#define FRONT_PAD(p) ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN)
75#define MIDDLE_PAD(p) (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN)
76#define DATA_PAD(p) (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN)
77
78#define CEPH_MSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
79
80static int do_recvmsg(struct socket *sock, struct iov_iter *it)
81{
82 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
83 int ret;
84
85 msg.msg_iter = *it;
86 while (iov_iter_count(i: it)) {
87 ret = sock_recvmsg(sock, msg: &msg, flags: msg.msg_flags);
88 if (ret <= 0) {
89 if (ret == -EAGAIN)
90 ret = 0;
91 return ret;
92 }
93
94 iov_iter_advance(i: it, bytes: ret);
95 }
96
97 WARN_ON(msg_data_left(&msg));
98 return 1;
99}
100
101/*
102 * Read as much as possible.
103 *
104 * Return:
105 * 1 - done, nothing (else) to read
106 * 0 - socket is empty, need to wait
107 * <0 - error
108 */
109static int ceph_tcp_recv(struct ceph_connection *con)
110{
111 int ret;
112
113 dout("%s con %p %s %zu\n", __func__, con,
114 iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need",
115 iov_iter_count(&con->v2.in_iter));
116 ret = do_recvmsg(sock: con->sock, it: &con->v2.in_iter);
117 dout("%s con %p ret %d left %zu\n", __func__, con, ret,
118 iov_iter_count(&con->v2.in_iter));
119 return ret;
120}
121
122static int do_sendmsg(struct socket *sock, struct iov_iter *it)
123{
124 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
125 int ret;
126
127 msg.msg_iter = *it;
128 while (iov_iter_count(i: it)) {
129 ret = sock_sendmsg(sock, msg: &msg);
130 if (ret <= 0) {
131 if (ret == -EAGAIN)
132 ret = 0;
133 return ret;
134 }
135
136 iov_iter_advance(i: it, bytes: ret);
137 }
138
139 WARN_ON(msg_data_left(&msg));
140 return 1;
141}
142
143static int do_try_sendpage(struct socket *sock, struct iov_iter *it)
144{
145 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
146 struct bio_vec bv;
147 int ret;
148
149 if (WARN_ON(!iov_iter_is_bvec(it)))
150 return -EINVAL;
151
152 while (iov_iter_count(i: it)) {
153 /* iov_iter_iovec() for ITER_BVEC */
154 bvec_set_page(bv: &bv, page: it->bvec->bv_page,
155 min(iov_iter_count(it),
156 it->bvec->bv_len - it->iov_offset),
157 offset: it->bvec->bv_offset + it->iov_offset);
158
159 /*
160 * MSG_SPLICE_PAGES cannot properly handle pages with
161 * page_count == 0, we need to fall back to sendmsg if
162 * that's the case.
163 *
164 * Same goes for slab pages: skb_can_coalesce() allows
165 * coalescing neighboring slab objects into a single frag
166 * which triggers one of hardened usercopy checks.
167 */
168 if (sendpage_ok(page: bv.bv_page))
169 msg.msg_flags |= MSG_SPLICE_PAGES;
170 else
171 msg.msg_flags &= ~MSG_SPLICE_PAGES;
172
173 iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: &bv, nr_segs: 1, count: bv.bv_len);
174 ret = sock_sendmsg(sock, msg: &msg);
175 if (ret <= 0) {
176 if (ret == -EAGAIN)
177 ret = 0;
178 return ret;
179 }
180
181 iov_iter_advance(i: it, bytes: ret);
182 }
183
184 return 1;
185}
186
187/*
188 * Write as much as possible. The socket is expected to be corked,
189 * so we don't bother with MSG_MORE here.
190 *
191 * Return:
192 * 1 - done, nothing (else) to write
193 * 0 - socket is full, need to wait
194 * <0 - error
195 */
196static int ceph_tcp_send(struct ceph_connection *con)
197{
198 int ret;
199
200 dout("%s con %p have %zu try_sendpage %d\n", __func__, con,
201 iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage);
202 if (con->v2.out_iter_sendpage)
203 ret = do_try_sendpage(sock: con->sock, it: &con->v2.out_iter);
204 else
205 ret = do_sendmsg(sock: con->sock, it: &con->v2.out_iter);
206 dout("%s con %p ret %d left %zu\n", __func__, con, ret,
207 iov_iter_count(&con->v2.out_iter));
208 return ret;
209}
210
211static void add_in_kvec(struct ceph_connection *con, void *buf, int len)
212{
213 BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs));
214 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
215
216 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf;
217 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len;
218 con->v2.in_kvec_cnt++;
219
220 con->v2.in_iter.nr_segs++;
221 con->v2.in_iter.count += len;
222}
223
224static void reset_in_kvecs(struct ceph_connection *con)
225{
226 WARN_ON(iov_iter_count(&con->v2.in_iter));
227
228 con->v2.in_kvec_cnt = 0;
229 iov_iter_kvec(i: &con->v2.in_iter, ITER_DEST, kvec: con->v2.in_kvecs, nr_segs: 0, count: 0);
230}
231
232static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv)
233{
234 WARN_ON(iov_iter_count(&con->v2.in_iter));
235
236 con->v2.in_bvec = *bv;
237 iov_iter_bvec(i: &con->v2.in_iter, ITER_DEST, bvec: &con->v2.in_bvec, nr_segs: 1, count: bv->bv_len);
238}
239
240static void set_in_skip(struct ceph_connection *con, int len)
241{
242 WARN_ON(iov_iter_count(&con->v2.in_iter));
243
244 dout("%s con %p len %d\n", __func__, con, len);
245 iov_iter_discard(i: &con->v2.in_iter, ITER_DEST, count: len);
246}
247
248static void add_out_kvec(struct ceph_connection *con, void *buf, int len)
249{
250 BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs));
251 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
252 WARN_ON(con->v2.out_zero);
253
254 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf;
255 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len;
256 con->v2.out_kvec_cnt++;
257
258 con->v2.out_iter.nr_segs++;
259 con->v2.out_iter.count += len;
260}
261
262static void reset_out_kvecs(struct ceph_connection *con)
263{
264 WARN_ON(iov_iter_count(&con->v2.out_iter));
265 WARN_ON(con->v2.out_zero);
266
267 con->v2.out_kvec_cnt = 0;
268
269 iov_iter_kvec(i: &con->v2.out_iter, ITER_SOURCE, kvec: con->v2.out_kvecs, nr_segs: 0, count: 0);
270 con->v2.out_iter_sendpage = false;
271}
272
273static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv,
274 bool zerocopy)
275{
276 WARN_ON(iov_iter_count(&con->v2.out_iter));
277 WARN_ON(con->v2.out_zero);
278
279 con->v2.out_bvec = *bv;
280 con->v2.out_iter_sendpage = zerocopy;
281 iov_iter_bvec(i: &con->v2.out_iter, ITER_SOURCE, bvec: &con->v2.out_bvec, nr_segs: 1,
282 count: con->v2.out_bvec.bv_len);
283}
284
285static void set_out_bvec_zero(struct ceph_connection *con)
286{
287 WARN_ON(iov_iter_count(&con->v2.out_iter));
288 WARN_ON(!con->v2.out_zero);
289
290 bvec_set_page(bv: &con->v2.out_bvec, page: ceph_zero_page,
291 min(con->v2.out_zero, (int)PAGE_SIZE), offset: 0);
292 con->v2.out_iter_sendpage = true;
293 iov_iter_bvec(i: &con->v2.out_iter, ITER_SOURCE, bvec: &con->v2.out_bvec, nr_segs: 1,
294 count: con->v2.out_bvec.bv_len);
295}
296
297static void out_zero_add(struct ceph_connection *con, int len)
298{
299 dout("%s con %p len %d\n", __func__, con, len);
300 con->v2.out_zero += len;
301}
302
303static void *alloc_conn_buf(struct ceph_connection *con, int len)
304{
305 void *buf;
306
307 dout("%s con %p len %d\n", __func__, con, len);
308
309 if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs)))
310 return NULL;
311
312 buf = kvmalloc(len, GFP_NOIO);
313 if (!buf)
314 return NULL;
315
316 con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf;
317 return buf;
318}
319
320static void free_conn_bufs(struct ceph_connection *con)
321{
322 while (con->v2.conn_buf_cnt)
323 kvfree(addr: con->v2.conn_bufs[--con->v2.conn_buf_cnt]);
324}
325
326static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len)
327{
328 BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs));
329
330 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf;
331 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len;
332 con->v2.in_sign_kvec_cnt++;
333}
334
335static void clear_in_sign_kvecs(struct ceph_connection *con)
336{
337 con->v2.in_sign_kvec_cnt = 0;
338}
339
340static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len)
341{
342 BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs));
343
344 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf;
345 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len;
346 con->v2.out_sign_kvec_cnt++;
347}
348
349static void clear_out_sign_kvecs(struct ceph_connection *con)
350{
351 con->v2.out_sign_kvec_cnt = 0;
352}
353
354static bool con_secure(struct ceph_connection *con)
355{
356 return con->v2.con_mode == CEPH_CON_MODE_SECURE;
357}
358
359static int front_len(const struct ceph_msg *msg)
360{
361 return le32_to_cpu(msg->hdr.front_len);
362}
363
364static int middle_len(const struct ceph_msg *msg)
365{
366 return le32_to_cpu(msg->hdr.middle_len);
367}
368
369static int data_len(const struct ceph_msg *msg)
370{
371 return le32_to_cpu(msg->hdr.data_len);
372}
373
374static bool need_padding(int len)
375{
376 return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN);
377}
378
379static int padded_len(int len)
380{
381 return ALIGN(len, CEPH_GCM_BLOCK_LEN);
382}
383
384static int padding_len(int len)
385{
386 return padded_len(len) - len;
387}
388
389/* preamble + control segment */
390static int head_onwire_len(int ctrl_len, bool secure)
391{
392 int head_len;
393 int rem_len;
394
395 BUG_ON(ctrl_len < 0 || ctrl_len > CEPH_MSG_MAX_CONTROL_LEN);
396
397 if (secure) {
398 head_len = CEPH_PREAMBLE_SECURE_LEN;
399 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
400 rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
401 head_len += padded_len(len: rem_len) + CEPH_GCM_TAG_LEN;
402 }
403 } else {
404 head_len = CEPH_PREAMBLE_PLAIN_LEN;
405 if (ctrl_len)
406 head_len += ctrl_len + CEPH_CRC_LEN;
407 }
408 return head_len;
409}
410
411/* front, middle and data segments + epilogue */
412static int __tail_onwire_len(int front_len, int middle_len, int data_len,
413 bool secure)
414{
415 BUG_ON(front_len < 0 || front_len > CEPH_MSG_MAX_FRONT_LEN ||
416 middle_len < 0 || middle_len > CEPH_MSG_MAX_MIDDLE_LEN ||
417 data_len < 0 || data_len > CEPH_MSG_MAX_DATA_LEN);
418
419 if (!front_len && !middle_len && !data_len)
420 return 0;
421
422 if (!secure)
423 return front_len + middle_len + data_len +
424 CEPH_EPILOGUE_PLAIN_LEN;
425
426 return padded_len(len: front_len) + padded_len(len: middle_len) +
427 padded_len(len: data_len) + CEPH_EPILOGUE_SECURE_LEN;
428}
429
430static int tail_onwire_len(const struct ceph_msg *msg, bool secure)
431{
432 return __tail_onwire_len(front_len: front_len(msg), middle_len: middle_len(msg),
433 data_len: data_len(msg), secure);
434}
435
436/* head_onwire_len(sizeof(struct ceph_msg_header2), false) */
437#define MESSAGE_HEAD_PLAIN_LEN (CEPH_PREAMBLE_PLAIN_LEN + \
438 sizeof(struct ceph_msg_header2) + \
439 CEPH_CRC_LEN)
440
441static const int frame_aligns[] = {
442 sizeof(void *),
443 sizeof(void *),
444 sizeof(void *),
445 PAGE_SIZE
446};
447
448/*
449 * Discards trailing empty segments, unless there is just one segment.
450 * A frame always has at least one (possibly empty) segment.
451 */
452static int calc_segment_count(const int *lens, int len_cnt)
453{
454 int i;
455
456 for (i = len_cnt - 1; i >= 0; i--) {
457 if (lens[i])
458 return i + 1;
459 }
460
461 return 1;
462}
463
464static void init_frame_desc(struct ceph_frame_desc *desc, int tag,
465 const int *lens, int len_cnt)
466{
467 int i;
468
469 memset(desc, 0, sizeof(*desc));
470
471 desc->fd_tag = tag;
472 desc->fd_seg_cnt = calc_segment_count(lens, len_cnt);
473 BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT);
474 for (i = 0; i < desc->fd_seg_cnt; i++) {
475 desc->fd_lens[i] = lens[i];
476 desc->fd_aligns[i] = frame_aligns[i];
477 }
478}
479
480/*
481 * Preamble crc covers everything up to itself (28 bytes) and
482 * is calculated and verified irrespective of the connection mode
483 * (i.e. even if the frame is encrypted).
484 */
485static void encode_preamble(const struct ceph_frame_desc *desc, void *p)
486{
487 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
488 void *start = p;
489 int i;
490
491 memset(p, 0, CEPH_PREAMBLE_LEN);
492
493 ceph_encode_8(p: &p, v: desc->fd_tag);
494 ceph_encode_8(p: &p, v: desc->fd_seg_cnt);
495 for (i = 0; i < desc->fd_seg_cnt; i++) {
496 ceph_encode_32(p: &p, v: desc->fd_lens[i]);
497 ceph_encode_16(p: &p, v: desc->fd_aligns[i]);
498 }
499
500 put_unaligned_le32(val: crc32c(crc: 0, p: start, len: crcp - start), p: crcp);
501}
502
503static int decode_preamble(void *p, struct ceph_frame_desc *desc)
504{
505 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
506 u32 crc, expected_crc;
507 int i;
508
509 crc = crc32c(crc: 0, p, len: crcp - p);
510 expected_crc = get_unaligned_le32(p: crcp);
511 if (crc != expected_crc) {
512 pr_err("bad preamble crc, calculated %u, expected %u\n",
513 crc, expected_crc);
514 return -EBADMSG;
515 }
516
517 memset(desc, 0, sizeof(*desc));
518
519 desc->fd_tag = ceph_decode_8(p: &p);
520 desc->fd_seg_cnt = ceph_decode_8(p: &p);
521 if (desc->fd_seg_cnt < 1 ||
522 desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) {
523 pr_err("bad segment count %d\n", desc->fd_seg_cnt);
524 return -EINVAL;
525 }
526 for (i = 0; i < desc->fd_seg_cnt; i++) {
527 desc->fd_lens[i] = ceph_decode_32(p: &p);
528 desc->fd_aligns[i] = ceph_decode_16(p: &p);
529 }
530
531 if (desc->fd_lens[0] < 0 ||
532 desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) {
533 pr_err("bad control segment length %d\n", desc->fd_lens[0]);
534 return -EINVAL;
535 }
536 if (desc->fd_lens[1] < 0 ||
537 desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) {
538 pr_err("bad front segment length %d\n", desc->fd_lens[1]);
539 return -EINVAL;
540 }
541 if (desc->fd_lens[2] < 0 ||
542 desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) {
543 pr_err("bad middle segment length %d\n", desc->fd_lens[2]);
544 return -EINVAL;
545 }
546 if (desc->fd_lens[3] < 0 ||
547 desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) {
548 pr_err("bad data segment length %d\n", desc->fd_lens[3]);
549 return -EINVAL;
550 }
551
552 /*
553 * This would fire for FRAME_TAG_WAIT (it has one empty
554 * segment), but we should never get it as client.
555 */
556 if (!desc->fd_lens[desc->fd_seg_cnt - 1]) {
557 pr_err("last segment empty, segment count %d\n",
558 desc->fd_seg_cnt);
559 return -EINVAL;
560 }
561
562 return 0;
563}
564
565static void encode_epilogue_plain(struct ceph_connection *con, bool aborted)
566{
567 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
568 FRAME_LATE_STATUS_COMPLETE;
569 cpu_to_le32s(&con->v2.out_epil.front_crc);
570 cpu_to_le32s(&con->v2.out_epil.middle_crc);
571 cpu_to_le32s(&con->v2.out_epil.data_crc);
572}
573
574static void encode_epilogue_secure(struct ceph_connection *con, bool aborted)
575{
576 memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil));
577 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
578 FRAME_LATE_STATUS_COMPLETE;
579}
580
581static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc,
582 u32 *data_crc)
583{
584 u8 late_status;
585
586 late_status = ceph_decode_8(p: &p);
587 if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) !=
588 FRAME_LATE_STATUS_COMPLETE) {
589 /* we should never get an aborted message as client */
590 pr_err("bad late_status 0x%x\n", late_status);
591 return -EINVAL;
592 }
593
594 if (front_crc && middle_crc && data_crc) {
595 *front_crc = ceph_decode_32(p: &p);
596 *middle_crc = ceph_decode_32(p: &p);
597 *data_crc = ceph_decode_32(p: &p);
598 }
599
600 return 0;
601}
602
603static void fill_header(struct ceph_msg_header *hdr,
604 const struct ceph_msg_header2 *hdr2,
605 int front_len, int middle_len, int data_len,
606 const struct ceph_entity_name *peer_name)
607{
608 hdr->seq = hdr2->seq;
609 hdr->tid = hdr2->tid;
610 hdr->type = hdr2->type;
611 hdr->priority = hdr2->priority;
612 hdr->version = hdr2->version;
613 hdr->front_len = cpu_to_le32(front_len);
614 hdr->middle_len = cpu_to_le32(middle_len);
615 hdr->data_len = cpu_to_le32(data_len);
616 hdr->data_off = hdr2->data_off;
617 hdr->src = *peer_name;
618 hdr->compat_version = hdr2->compat_version;
619 hdr->reserved = 0;
620 hdr->crc = 0;
621}
622
623static void fill_header2(struct ceph_msg_header2 *hdr2,
624 const struct ceph_msg_header *hdr, u64 ack_seq)
625{
626 hdr2->seq = hdr->seq;
627 hdr2->tid = hdr->tid;
628 hdr2->type = hdr->type;
629 hdr2->priority = hdr->priority;
630 hdr2->version = hdr->version;
631 hdr2->data_pre_padding_len = 0;
632 hdr2->data_off = hdr->data_off;
633 hdr2->ack_seq = cpu_to_le64(ack_seq);
634 hdr2->flags = 0;
635 hdr2->compat_version = hdr->compat_version;
636 hdr2->reserved = 0;
637}
638
639static int verify_control_crc(struct ceph_connection *con)
640{
641 int ctrl_len = con->v2.in_desc.fd_lens[0];
642 u32 crc, expected_crc;
643
644 WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len);
645 WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN);
646
647 crc = crc32c(crc: -1, p: con->v2.in_kvecs[0].iov_base, len: ctrl_len);
648 expected_crc = get_unaligned_le32(p: con->v2.in_kvecs[1].iov_base);
649 if (crc != expected_crc) {
650 pr_err("bad control crc, calculated %u, expected %u\n",
651 crc, expected_crc);
652 return -EBADMSG;
653 }
654
655 return 0;
656}
657
658static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc,
659 u32 middle_crc, u32 data_crc)
660{
661 if (front_len(msg: con->in_msg)) {
662 con->in_front_crc = crc32c(crc: -1, p: con->in_msg->front.iov_base,
663 len: front_len(msg: con->in_msg));
664 } else {
665 WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg));
666 con->in_front_crc = -1;
667 }
668
669 if (middle_len(msg: con->in_msg))
670 con->in_middle_crc = crc32c(crc: -1,
671 p: con->in_msg->middle->vec.iov_base,
672 len: middle_len(msg: con->in_msg));
673 else if (data_len(msg: con->in_msg))
674 con->in_middle_crc = -1;
675 else
676 con->in_middle_crc = 0;
677
678 if (!data_len(msg: con->in_msg))
679 con->in_data_crc = 0;
680
681 dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg,
682 con->in_front_crc, con->in_middle_crc, con->in_data_crc);
683
684 if (con->in_front_crc != front_crc) {
685 pr_err("bad front crc, calculated %u, expected %u\n",
686 con->in_front_crc, front_crc);
687 return -EBADMSG;
688 }
689 if (con->in_middle_crc != middle_crc) {
690 pr_err("bad middle crc, calculated %u, expected %u\n",
691 con->in_middle_crc, middle_crc);
692 return -EBADMSG;
693 }
694 if (con->in_data_crc != data_crc) {
695 pr_err("bad data crc, calculated %u, expected %u\n",
696 con->in_data_crc, data_crc);
697 return -EBADMSG;
698 }
699
700 return 0;
701}
702
703static int setup_crypto(struct ceph_connection *con,
704 const u8 *session_key, int session_key_len,
705 const u8 *con_secret, int con_secret_len)
706{
707 unsigned int noio_flag;
708 int ret;
709
710 dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n",
711 __func__, con, con->v2.con_mode, session_key_len, con_secret_len);
712 WARN_ON(con->v2.hmac_key_set || con->v2.gcm_tfm || con->v2.gcm_req);
713
714 if (con->v2.con_mode != CEPH_CON_MODE_CRC &&
715 con->v2.con_mode != CEPH_CON_MODE_SECURE) {
716 pr_err("bad con_mode %d\n", con->v2.con_mode);
717 return -EINVAL;
718 }
719
720 if (!session_key_len) {
721 WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC);
722 WARN_ON(con_secret_len);
723 return 0; /* auth_none */
724 }
725
726 hmac_sha256_preparekey(key: &con->v2.hmac_key, raw_key: session_key, raw_key_len: session_key_len);
727 con->v2.hmac_key_set = true;
728
729 if (con->v2.con_mode == CEPH_CON_MODE_CRC) {
730 WARN_ON(con_secret_len);
731 return 0; /* auth_x, plain mode */
732 }
733
734 if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) {
735 pr_err("con_secret too small %d\n", con_secret_len);
736 return -EINVAL;
737 }
738
739 noio_flag = memalloc_noio_save();
740 con->v2.gcm_tfm = crypto_alloc_aead(alg_name: "gcm(aes)", type: 0, mask: 0);
741 memalloc_noio_restore(flags: noio_flag);
742 if (IS_ERR(ptr: con->v2.gcm_tfm)) {
743 ret = PTR_ERR(ptr: con->v2.gcm_tfm);
744 con->v2.gcm_tfm = NULL;
745 pr_err("failed to allocate gcm tfm context: %d\n", ret);
746 return ret;
747 }
748
749 WARN_ON((unsigned long)con_secret &
750 crypto_aead_alignmask(con->v2.gcm_tfm));
751 ret = crypto_aead_setkey(tfm: con->v2.gcm_tfm, key: con_secret, CEPH_GCM_KEY_LEN);
752 if (ret) {
753 pr_err("failed to set gcm key: %d\n", ret);
754 return ret;
755 }
756
757 WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN);
758 ret = crypto_aead_setauthsize(tfm: con->v2.gcm_tfm, CEPH_GCM_TAG_LEN);
759 if (ret) {
760 pr_err("failed to set gcm tag size: %d\n", ret);
761 return ret;
762 }
763
764 con->v2.gcm_req = aead_request_alloc(tfm: con->v2.gcm_tfm, GFP_NOIO);
765 if (!con->v2.gcm_req) {
766 pr_err("failed to allocate gcm request\n");
767 return -ENOMEM;
768 }
769
770 crypto_init_wait(wait: &con->v2.gcm_wait);
771 aead_request_set_callback(req: con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
772 compl: crypto_req_done, data: &con->v2.gcm_wait);
773
774 memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN,
775 CEPH_GCM_IV_LEN);
776 memcpy(&con->v2.out_gcm_nonce,
777 con_secret + CEPH_GCM_KEY_LEN + CEPH_GCM_IV_LEN,
778 CEPH_GCM_IV_LEN);
779 return 0; /* auth_x, secure mode */
780}
781
782static void ceph_hmac_sha256(struct ceph_connection *con,
783 const struct kvec *kvecs, int kvec_cnt,
784 u8 hmac[SHA256_DIGEST_SIZE])
785{
786 struct hmac_sha256_ctx ctx;
787 int i;
788
789 dout("%s con %p hmac_key_set %d kvec_cnt %d\n", __func__, con,
790 con->v2.hmac_key_set, kvec_cnt);
791
792 if (!con->v2.hmac_key_set) {
793 memset(hmac, 0, SHA256_DIGEST_SIZE);
794 return; /* auth_none */
795 }
796
797 /* auth_x, both plain and secure modes */
798 hmac_sha256_init(ctx: &ctx, key: &con->v2.hmac_key);
799 for (i = 0; i < kvec_cnt; i++)
800 hmac_sha256_update(ctx: &ctx, data: kvecs[i].iov_base, data_len: kvecs[i].iov_len);
801 hmac_sha256_final(ctx: &ctx, out: hmac);
802}
803
804static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce)
805{
806 u64 counter;
807
808 counter = le64_to_cpu(nonce->counter);
809 nonce->counter = cpu_to_le64(counter + 1);
810}
811
812static int gcm_crypt(struct ceph_connection *con, bool encrypt,
813 struct scatterlist *src, struct scatterlist *dst,
814 int src_len)
815{
816 struct ceph_gcm_nonce *nonce;
817 int ret;
818
819 nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce;
820
821 aead_request_set_ad(req: con->v2.gcm_req, assoclen: 0); /* no AAD */
822 aead_request_set_crypt(req: con->v2.gcm_req, src, dst, cryptlen: src_len, iv: (u8 *)nonce);
823 ret = crypto_wait_req(err: encrypt ? crypto_aead_encrypt(req: con->v2.gcm_req) :
824 crypto_aead_decrypt(req: con->v2.gcm_req),
825 wait: &con->v2.gcm_wait);
826 if (ret)
827 return ret;
828
829 gcm_inc_nonce(nonce);
830 return 0;
831}
832
833static void get_bvec_at(struct ceph_msg_data_cursor *cursor,
834 struct bio_vec *bv)
835{
836 struct page *page;
837 size_t off, len;
838
839 WARN_ON(!cursor->total_resid);
840
841 /* skip zero-length data items */
842 while (!cursor->resid)
843 ceph_msg_data_advance(cursor, bytes: 0);
844
845 /* get a piece of data, cursor isn't advanced */
846 page = ceph_msg_data_next(cursor, page_offset: &off, length: &len);
847 bvec_set_page(bv, page, len, offset: off);
848}
849
850static int calc_sg_cnt(void *buf, int buf_len)
851{
852 int sg_cnt;
853
854 if (!buf_len)
855 return 0;
856
857 sg_cnt = need_padding(len: buf_len) ? 1 : 0;
858 if (is_vmalloc_addr(x: buf)) {
859 WARN_ON(offset_in_page(buf));
860 sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT;
861 } else {
862 sg_cnt++;
863 }
864
865 return sg_cnt;
866}
867
868static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor)
869{
870 int data_len = cursor->total_resid;
871 struct bio_vec bv;
872 int sg_cnt;
873
874 if (!data_len)
875 return 0;
876
877 sg_cnt = need_padding(len: data_len) ? 1 : 0;
878 do {
879 get_bvec_at(cursor, bv: &bv);
880 sg_cnt++;
881
882 ceph_msg_data_advance(cursor, bytes: bv.bv_len);
883 } while (cursor->total_resid);
884
885 return sg_cnt;
886}
887
888static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad)
889{
890 void *end = buf + buf_len;
891 struct page *page;
892 int len;
893 void *p;
894
895 if (!buf_len)
896 return;
897
898 if (is_vmalloc_addr(x: buf)) {
899 p = buf;
900 do {
901 page = vmalloc_to_page(addr: p);
902 len = min_t(int, end - p, PAGE_SIZE);
903 WARN_ON(!page || !len || offset_in_page(p));
904 sg_set_page(sg: *sg, page, len, offset: 0);
905 *sg = sg_next(sg: *sg);
906 p += len;
907 } while (p != end);
908 } else {
909 sg_set_buf(sg: *sg, buf, buflen: buf_len);
910 *sg = sg_next(sg: *sg);
911 }
912
913 if (need_padding(len: buf_len)) {
914 sg_set_buf(sg: *sg, buf: pad, buflen: padding_len(len: buf_len));
915 *sg = sg_next(sg: *sg);
916 }
917}
918
919static void init_sgs_cursor(struct scatterlist **sg,
920 struct ceph_msg_data_cursor *cursor, u8 *pad)
921{
922 int data_len = cursor->total_resid;
923 struct bio_vec bv;
924
925 if (!data_len)
926 return;
927
928 do {
929 get_bvec_at(cursor, bv: &bv);
930 sg_set_page(sg: *sg, page: bv.bv_page, len: bv.bv_len, offset: bv.bv_offset);
931 *sg = sg_next(sg: *sg);
932
933 ceph_msg_data_advance(cursor, bytes: bv.bv_len);
934 } while (cursor->total_resid);
935
936 if (need_padding(len: data_len)) {
937 sg_set_buf(sg: *sg, buf: pad, buflen: padding_len(len: data_len));
938 *sg = sg_next(sg: *sg);
939 }
940}
941
942/**
943 * init_sgs_pages: set up scatterlist on an array of page pointers
944 * @sg: scatterlist to populate
945 * @pages: pointer to page array
946 * @dpos: position in the array to start (bytes)
947 * @dlen: len to add to sg (bytes)
948 * @pad: pointer to pad destination (if any)
949 *
950 * Populate the scatterlist from the page array, starting at an arbitrary
951 * byte in the array and running for a specified length.
952 */
953static void init_sgs_pages(struct scatterlist **sg, struct page **pages,
954 int dpos, int dlen, u8 *pad)
955{
956 int idx = dpos >> PAGE_SHIFT;
957 int off = offset_in_page(dpos);
958 int resid = dlen;
959
960 do {
961 int len = min(resid, (int)PAGE_SIZE - off);
962
963 sg_set_page(sg: *sg, page: pages[idx], len, offset: off);
964 *sg = sg_next(sg: *sg);
965 off = 0;
966 ++idx;
967 resid -= len;
968 } while (resid);
969
970 if (need_padding(len: dlen)) {
971 sg_set_buf(sg: *sg, buf: pad, buflen: padding_len(len: dlen));
972 *sg = sg_next(sg: *sg);
973 }
974}
975
976static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg,
977 u8 *front_pad, u8 *middle_pad, u8 *data_pad,
978 void *epilogue, struct page **pages, int dpos,
979 bool add_tag)
980{
981 struct ceph_msg_data_cursor cursor;
982 struct scatterlist *cur_sg;
983 int dlen = data_len(msg);
984 int sg_cnt;
985 int ret;
986
987 if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
988 return 0;
989
990 sg_cnt = 1; /* epilogue + [auth tag] */
991 if (front_len(msg))
992 sg_cnt += calc_sg_cnt(buf: msg->front.iov_base,
993 buf_len: front_len(msg));
994 if (middle_len(msg))
995 sg_cnt += calc_sg_cnt(buf: msg->middle->vec.iov_base,
996 buf_len: middle_len(msg));
997 if (dlen) {
998 if (pages) {
999 sg_cnt += calc_pages_for(off: dpos, len: dlen);
1000 if (need_padding(len: dlen))
1001 sg_cnt++;
1002 } else {
1003 ceph_msg_data_cursor_init(cursor: &cursor, msg, length: dlen);
1004 sg_cnt += calc_sg_cnt_cursor(cursor: &cursor);
1005 }
1006 }
1007
1008 ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO);
1009 if (ret)
1010 return ret;
1011
1012 cur_sg = sgt->sgl;
1013 if (front_len(msg))
1014 init_sgs(sg: &cur_sg, buf: msg->front.iov_base, buf_len: front_len(msg),
1015 pad: front_pad);
1016 if (middle_len(msg))
1017 init_sgs(sg: &cur_sg, buf: msg->middle->vec.iov_base, buf_len: middle_len(msg),
1018 pad: middle_pad);
1019 if (dlen) {
1020 if (pages) {
1021 init_sgs_pages(sg: &cur_sg, pages, dpos, dlen, pad: data_pad);
1022 } else {
1023 ceph_msg_data_cursor_init(cursor: &cursor, msg, length: dlen);
1024 init_sgs_cursor(sg: &cur_sg, cursor: &cursor, pad: data_pad);
1025 }
1026 }
1027
1028 WARN_ON(!sg_is_last(cur_sg));
1029 sg_set_buf(sg: cur_sg, buf: epilogue,
1030 CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0));
1031 return 0;
1032}
1033
1034static int decrypt_preamble(struct ceph_connection *con)
1035{
1036 struct scatterlist sg;
1037
1038 sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN);
1039 return gcm_crypt(con, encrypt: false, src: &sg, dst: &sg, CEPH_PREAMBLE_SECURE_LEN);
1040}
1041
1042static int decrypt_control_remainder(struct ceph_connection *con)
1043{
1044 int ctrl_len = con->v2.in_desc.fd_lens[0];
1045 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1046 int pt_len = padding_len(len: rem_len) + CEPH_GCM_TAG_LEN;
1047 struct scatterlist sgs[2];
1048
1049 WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len);
1050 WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len);
1051
1052 sg_init_table(sgs, 2);
1053 sg_set_buf(sg: &sgs[0], buf: con->v2.in_kvecs[0].iov_base, buflen: rem_len);
1054 sg_set_buf(sg: &sgs[1], buf: con->v2.in_buf, buflen: pt_len);
1055
1056 return gcm_crypt(con, encrypt: false, src: sgs, dst: sgs,
1057 src_len: padded_len(len: rem_len) + CEPH_GCM_TAG_LEN);
1058}
1059
1060/* Process sparse read data that lives in a buffer */
1061static int process_v2_sparse_read(struct ceph_connection *con,
1062 struct page **pages, int spos)
1063{
1064 struct ceph_msg_data_cursor cursor;
1065 int ret;
1066
1067 ceph_msg_data_cursor_init(cursor: &cursor, msg: con->in_msg,
1068 length: con->in_msg->sparse_read_total);
1069
1070 for (;;) {
1071 char *buf = NULL;
1072
1073 ret = con->ops->sparse_read(con, &cursor, &buf);
1074 if (ret <= 0)
1075 return ret;
1076
1077 dout("%s: sparse_read return %x buf %p\n", __func__, ret, buf);
1078
1079 do {
1080 int idx = spos >> PAGE_SHIFT;
1081 int soff = offset_in_page(spos);
1082 struct page *spage = con->v2.in_enc_pages[idx];
1083 int len = min_t(int, ret, PAGE_SIZE - soff);
1084
1085 if (buf) {
1086 memcpy_from_page(to: buf, page: spage, offset: soff, len);
1087 buf += len;
1088 } else {
1089 struct bio_vec bv;
1090
1091 get_bvec_at(cursor: &cursor, bv: &bv);
1092 len = min_t(int, len, bv.bv_len);
1093 memcpy_page(dst_page: bv.bv_page, dst_off: bv.bv_offset,
1094 src_page: spage, src_off: soff, len);
1095 ceph_msg_data_advance(cursor: &cursor, bytes: len);
1096 }
1097 spos += len;
1098 ret -= len;
1099 } while (ret);
1100 }
1101}
1102
1103static int decrypt_tail(struct ceph_connection *con)
1104{
1105 struct sg_table enc_sgt = {};
1106 struct sg_table sgt = {};
1107 struct page **pages = NULL;
1108 bool sparse = !!con->in_msg->sparse_read_total;
1109 int dpos = 0;
1110 int tail_len;
1111 int ret;
1112
1113 tail_len = tail_onwire_len(msg: con->in_msg, secure: true);
1114 ret = sg_alloc_table_from_pages(sgt: &enc_sgt, pages: con->v2.in_enc_pages,
1115 n_pages: con->v2.in_enc_page_cnt, offset: 0, size: tail_len,
1116 GFP_NOIO);
1117 if (ret)
1118 goto out;
1119
1120 if (sparse) {
1121 dpos = padded_len(len: front_len(msg: con->in_msg) + padded_len(len: middle_len(msg: con->in_msg)));
1122 pages = con->v2.in_enc_pages;
1123 }
1124
1125 ret = setup_message_sgs(sgt: &sgt, msg: con->in_msg, FRONT_PAD(con->v2.in_buf),
1126 MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf),
1127 epilogue: con->v2.in_buf, pages, dpos, add_tag: true);
1128 if (ret)
1129 goto out;
1130
1131 dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con,
1132 con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents);
1133 ret = gcm_crypt(con, encrypt: false, src: enc_sgt.sgl, dst: sgt.sgl, src_len: tail_len);
1134 if (ret)
1135 goto out;
1136
1137 if (sparse && data_len(msg: con->in_msg)) {
1138 ret = process_v2_sparse_read(con, pages: con->v2.in_enc_pages, spos: dpos);
1139 if (ret)
1140 goto out;
1141 }
1142
1143 WARN_ON(!con->v2.in_enc_page_cnt);
1144 ceph_release_page_vector(pages: con->v2.in_enc_pages,
1145 num_pages: con->v2.in_enc_page_cnt);
1146 con->v2.in_enc_pages = NULL;
1147 con->v2.in_enc_page_cnt = 0;
1148
1149out:
1150 sg_free_table(&sgt);
1151 sg_free_table(&enc_sgt);
1152 return ret;
1153}
1154
1155static int prepare_banner(struct ceph_connection *con)
1156{
1157 int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8;
1158 void *buf, *p;
1159
1160 buf = alloc_conn_buf(con, len: buf_len);
1161 if (!buf)
1162 return -ENOMEM;
1163
1164 p = buf;
1165 ceph_encode_copy(p: &p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN);
1166 ceph_encode_16(p: &p, v: sizeof(u64) + sizeof(u64));
1167 ceph_encode_64(p: &p, CEPH_MSGR2_SUPPORTED_FEATURES);
1168 ceph_encode_64(p: &p, CEPH_MSGR2_REQUIRED_FEATURES);
1169 WARN_ON(p != buf + buf_len);
1170
1171 add_out_kvec(con, buf, len: buf_len);
1172 add_out_sign_kvec(con, buf, len: buf_len);
1173 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1174 return 0;
1175}
1176
1177/*
1178 * base:
1179 * preamble
1180 * control body (ctrl_len bytes)
1181 * space for control crc
1182 *
1183 * extdata (optional):
1184 * control body (extdata_len bytes)
1185 *
1186 * Compute control crc and gather base and extdata into:
1187 *
1188 * preamble
1189 * control body (ctrl_len + extdata_len bytes)
1190 * control crc
1191 *
1192 * Preamble should already be encoded at the start of base.
1193 */
1194static void prepare_head_plain(struct ceph_connection *con, void *base,
1195 int ctrl_len, void *extdata, int extdata_len,
1196 bool to_be_signed)
1197{
1198 int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN;
1199 void *crcp = base + base_len - CEPH_CRC_LEN;
1200 u32 crc;
1201
1202 crc = crc32c(crc: -1, CTRL_BODY(base), len: ctrl_len);
1203 if (extdata_len)
1204 crc = crc32c(crc, p: extdata, len: extdata_len);
1205 put_unaligned_le32(val: crc, p: crcp);
1206
1207 if (!extdata_len) {
1208 add_out_kvec(con, buf: base, len: base_len);
1209 if (to_be_signed)
1210 add_out_sign_kvec(con, buf: base, len: base_len);
1211 return;
1212 }
1213
1214 add_out_kvec(con, buf: base, len: crcp - base);
1215 add_out_kvec(con, buf: extdata, len: extdata_len);
1216 add_out_kvec(con, buf: crcp, CEPH_CRC_LEN);
1217 if (to_be_signed) {
1218 add_out_sign_kvec(con, buf: base, len: crcp - base);
1219 add_out_sign_kvec(con, buf: extdata, len: extdata_len);
1220 add_out_sign_kvec(con, buf: crcp, CEPH_CRC_LEN);
1221 }
1222}
1223
1224static int prepare_head_secure_small(struct ceph_connection *con,
1225 void *base, int ctrl_len)
1226{
1227 struct scatterlist sg;
1228 int ret;
1229
1230 /* inline buffer padding? */
1231 if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN)
1232 memset(CTRL_BODY(base) + ctrl_len, 0,
1233 CEPH_PREAMBLE_INLINE_LEN - ctrl_len);
1234
1235 sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN);
1236 ret = gcm_crypt(con, encrypt: true, src: &sg, dst: &sg,
1237 CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN);
1238 if (ret)
1239 return ret;
1240
1241 add_out_kvec(con, buf: base, CEPH_PREAMBLE_SECURE_LEN);
1242 return 0;
1243}
1244
1245/*
1246 * base:
1247 * preamble
1248 * control body (ctrl_len bytes)
1249 * space for padding, if needed
1250 * space for control remainder auth tag
1251 * space for preamble auth tag
1252 *
1253 * Encrypt preamble and the inline portion, then encrypt the remainder
1254 * and gather into:
1255 *
1256 * preamble
1257 * control body (48 bytes)
1258 * preamble auth tag
1259 * control body (ctrl_len - 48 bytes)
1260 * zero padding, if needed
1261 * control remainder auth tag
1262 *
1263 * Preamble should already be encoded at the start of base.
1264 */
1265static int prepare_head_secure_big(struct ceph_connection *con,
1266 void *base, int ctrl_len)
1267{
1268 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1269 void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN;
1270 void *rem_tag = rem + padded_len(len: rem_len);
1271 void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN;
1272 struct scatterlist sgs[2];
1273 int ret;
1274
1275 sg_init_table(sgs, 2);
1276 sg_set_buf(sg: &sgs[0], buf: base, buflen: rem - base);
1277 sg_set_buf(sg: &sgs[1], buf: pmbl_tag, CEPH_GCM_TAG_LEN);
1278 ret = gcm_crypt(con, encrypt: true, src: sgs, dst: sgs, src_len: rem - base);
1279 if (ret)
1280 return ret;
1281
1282 /* control remainder padding? */
1283 if (need_padding(len: rem_len))
1284 memset(rem + rem_len, 0, padding_len(rem_len));
1285
1286 sg_init_one(&sgs[0], rem, pmbl_tag - rem);
1287 ret = gcm_crypt(con, encrypt: true, src: sgs, dst: sgs, src_len: rem_tag - rem);
1288 if (ret)
1289 return ret;
1290
1291 add_out_kvec(con, buf: base, len: rem - base);
1292 add_out_kvec(con, buf: pmbl_tag, CEPH_GCM_TAG_LEN);
1293 add_out_kvec(con, buf: rem, len: pmbl_tag - rem);
1294 return 0;
1295}
1296
1297static int __prepare_control(struct ceph_connection *con, int tag,
1298 void *base, int ctrl_len, void *extdata,
1299 int extdata_len, bool to_be_signed)
1300{
1301 int total_len = ctrl_len + extdata_len;
1302 struct ceph_frame_desc desc;
1303 int ret;
1304
1305 dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag,
1306 total_len, ctrl_len, extdata_len);
1307
1308 /* extdata may be vmalloc'ed but not base */
1309 if (WARN_ON(is_vmalloc_addr(base) || !ctrl_len))
1310 return -EINVAL;
1311
1312 init_frame_desc(desc: &desc, tag, lens: &total_len, len_cnt: 1);
1313 encode_preamble(desc: &desc, p: base);
1314
1315 if (con_secure(con)) {
1316 if (WARN_ON(extdata_len || to_be_signed))
1317 return -EINVAL;
1318
1319 if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN)
1320 /* fully inlined, inline buffer may need padding */
1321 ret = prepare_head_secure_small(con, base, ctrl_len);
1322 else
1323 /* partially inlined, inline buffer is full */
1324 ret = prepare_head_secure_big(con, base, ctrl_len);
1325 if (ret)
1326 return ret;
1327 } else {
1328 prepare_head_plain(con, base, ctrl_len, extdata, extdata_len,
1329 to_be_signed);
1330 }
1331
1332 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1333 return 0;
1334}
1335
1336static int prepare_control(struct ceph_connection *con, int tag,
1337 void *base, int ctrl_len)
1338{
1339 return __prepare_control(con, tag, base, ctrl_len, NULL, extdata_len: 0, to_be_signed: false);
1340}
1341
1342static int prepare_hello(struct ceph_connection *con)
1343{
1344 void *buf, *p;
1345 int ctrl_len;
1346
1347 ctrl_len = 1 + ceph_entity_addr_encoding_len(addr: &con->peer_addr);
1348 buf = alloc_conn_buf(con, len: head_onwire_len(ctrl_len, secure: false));
1349 if (!buf)
1350 return -ENOMEM;
1351
1352 p = CTRL_BODY(buf);
1353 ceph_encode_8(p: &p, CEPH_ENTITY_TYPE_CLIENT);
1354 ceph_encode_entity_addr(p: &p, addr: &con->peer_addr);
1355 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1356
1357 return __prepare_control(con, FRAME_TAG_HELLO, base: buf, ctrl_len,
1358 NULL, extdata_len: 0, to_be_signed: true);
1359}
1360
1361/* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */
1362#define AUTH_BUF_LEN (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN)
1363
1364static int prepare_auth_request(struct ceph_connection *con)
1365{
1366 void *authorizer, *authorizer_copy;
1367 int ctrl_len, authorizer_len;
1368 void *buf;
1369 int ret;
1370
1371 ctrl_len = AUTH_BUF_LEN;
1372 buf = alloc_conn_buf(con, len: head_onwire_len(ctrl_len, secure: false));
1373 if (!buf)
1374 return -ENOMEM;
1375
1376 mutex_unlock(lock: &con->mutex);
1377 ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len,
1378 &authorizer, &authorizer_len);
1379 mutex_lock(&con->mutex);
1380 if (con->state != CEPH_CON_S_V2_HELLO) {
1381 dout("%s con %p state changed to %d\n", __func__, con,
1382 con->state);
1383 return -EAGAIN;
1384 }
1385
1386 dout("%s con %p get_auth_request ret %d\n", __func__, con, ret);
1387 if (ret)
1388 return ret;
1389
1390 authorizer_copy = alloc_conn_buf(con, len: authorizer_len);
1391 if (!authorizer_copy)
1392 return -ENOMEM;
1393
1394 memcpy(authorizer_copy, authorizer, authorizer_len);
1395
1396 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, base: buf, ctrl_len,
1397 extdata: authorizer_copy, extdata_len: authorizer_len, to_be_signed: true);
1398}
1399
1400static int prepare_auth_request_more(struct ceph_connection *con,
1401 void *reply, int reply_len)
1402{
1403 int ctrl_len, authorizer_len;
1404 void *authorizer;
1405 void *buf;
1406 int ret;
1407
1408 ctrl_len = AUTH_BUF_LEN;
1409 buf = alloc_conn_buf(con, len: head_onwire_len(ctrl_len, secure: false));
1410 if (!buf)
1411 return -ENOMEM;
1412
1413 mutex_unlock(lock: &con->mutex);
1414 ret = con->ops->handle_auth_reply_more(con, reply, reply_len,
1415 CTRL_BODY(buf), &ctrl_len,
1416 &authorizer, &authorizer_len);
1417 mutex_lock(&con->mutex);
1418 if (con->state != CEPH_CON_S_V2_AUTH) {
1419 dout("%s con %p state changed to %d\n", __func__, con,
1420 con->state);
1421 return -EAGAIN;
1422 }
1423
1424 dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret);
1425 if (ret)
1426 return ret;
1427
1428 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, base: buf,
1429 ctrl_len, extdata: authorizer, extdata_len: authorizer_len, to_be_signed: true);
1430}
1431
1432static int prepare_auth_signature(struct ceph_connection *con)
1433{
1434 void *buf;
1435
1436 buf = alloc_conn_buf(con, len: head_onwire_len(SHA256_DIGEST_SIZE,
1437 secure: con_secure(con)));
1438 if (!buf)
1439 return -ENOMEM;
1440
1441 ceph_hmac_sha256(con, kvecs: con->v2.in_sign_kvecs, kvec_cnt: con->v2.in_sign_kvec_cnt,
1442 CTRL_BODY(buf));
1443
1444 return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, base: buf,
1445 SHA256_DIGEST_SIZE);
1446}
1447
1448static int prepare_client_ident(struct ceph_connection *con)
1449{
1450 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1451 struct ceph_client *client = from_msgr(con->msgr);
1452 u64 global_id = ceph_client_gid(client);
1453 void *buf, *p;
1454 int ctrl_len;
1455
1456 WARN_ON(con->v2.server_cookie);
1457 WARN_ON(con->v2.connect_seq);
1458 WARN_ON(con->v2.peer_global_seq);
1459
1460 if (!con->v2.client_cookie) {
1461 do {
1462 get_random_bytes(buf: &con->v2.client_cookie,
1463 len: sizeof(con->v2.client_cookie));
1464 } while (!con->v2.client_cookie);
1465 dout("%s con %p generated cookie 0x%llx\n", __func__, con,
1466 con->v2.client_cookie);
1467 } else {
1468 dout("%s con %p cookie already set 0x%llx\n", __func__, con,
1469 con->v2.client_cookie);
1470 }
1471
1472 dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n",
1473 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1474 ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce),
1475 global_id, con->v2.global_seq, client->supported_features,
1476 client->required_features, con->v2.client_cookie);
1477
1478 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(addr: my_addr) +
1479 ceph_entity_addr_encoding_len(addr: &con->peer_addr) + 6 * 8;
1480 buf = alloc_conn_buf(con, len: head_onwire_len(ctrl_len, secure: con_secure(con)));
1481 if (!buf)
1482 return -ENOMEM;
1483
1484 p = CTRL_BODY(buf);
1485 ceph_encode_8(p: &p, v: 2); /* addrvec marker */
1486 ceph_encode_32(p: &p, v: 1); /* addr_cnt */
1487 ceph_encode_entity_addr(p: &p, addr: my_addr);
1488 ceph_encode_entity_addr(p: &p, addr: &con->peer_addr);
1489 ceph_encode_64(p: &p, v: global_id);
1490 ceph_encode_64(p: &p, v: con->v2.global_seq);
1491 ceph_encode_64(p: &p, v: client->supported_features);
1492 ceph_encode_64(p: &p, v: client->required_features);
1493 ceph_encode_64(p: &p, v: 0); /* flags */
1494 ceph_encode_64(p: &p, v: con->v2.client_cookie);
1495 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1496
1497 return prepare_control(con, FRAME_TAG_CLIENT_IDENT, base: buf, ctrl_len);
1498}
1499
1500static int prepare_session_reconnect(struct ceph_connection *con)
1501{
1502 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1503 void *buf, *p;
1504 int ctrl_len;
1505
1506 WARN_ON(!con->v2.client_cookie);
1507 WARN_ON(!con->v2.server_cookie);
1508 WARN_ON(!con->v2.connect_seq);
1509 WARN_ON(!con->v2.peer_global_seq);
1510
1511 dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n",
1512 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1513 con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq,
1514 con->v2.connect_seq, con->in_seq);
1515
1516 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(addr: my_addr) + 5 * 8;
1517 buf = alloc_conn_buf(con, len: head_onwire_len(ctrl_len, secure: con_secure(con)));
1518 if (!buf)
1519 return -ENOMEM;
1520
1521 p = CTRL_BODY(buf);
1522 ceph_encode_8(p: &p, v: 2); /* entity_addrvec_t marker */
1523 ceph_encode_32(p: &p, v: 1); /* my_addrs len */
1524 ceph_encode_entity_addr(p: &p, addr: my_addr);
1525 ceph_encode_64(p: &p, v: con->v2.client_cookie);
1526 ceph_encode_64(p: &p, v: con->v2.server_cookie);
1527 ceph_encode_64(p: &p, v: con->v2.global_seq);
1528 ceph_encode_64(p: &p, v: con->v2.connect_seq);
1529 ceph_encode_64(p: &p, v: con->in_seq);
1530 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1531
1532 return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, base: buf, ctrl_len);
1533}
1534
1535static int prepare_keepalive2(struct ceph_connection *con)
1536{
1537 struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf);
1538 struct timespec64 now;
1539
1540 ktime_get_real_ts64(tv: &now);
1541 dout("%s con %p timestamp %ptSp\n", __func__, con, &now);
1542
1543 ceph_encode_timespec64(tv: ts, ts: &now);
1544
1545 reset_out_kvecs(con);
1546 return prepare_control(con, FRAME_TAG_KEEPALIVE2, base: con->v2.out_buf,
1547 ctrl_len: sizeof(struct ceph_timespec));
1548}
1549
1550static int prepare_ack(struct ceph_connection *con)
1551{
1552 void *p;
1553
1554 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1555 con->in_seq_acked, con->in_seq);
1556 con->in_seq_acked = con->in_seq;
1557
1558 p = CTRL_BODY(con->v2.out_buf);
1559 ceph_encode_64(p: &p, v: con->in_seq_acked);
1560
1561 reset_out_kvecs(con);
1562 return prepare_control(con, FRAME_TAG_ACK, base: con->v2.out_buf, ctrl_len: 8);
1563}
1564
1565static void prepare_epilogue_plain(struct ceph_connection *con,
1566 struct ceph_msg *msg, bool aborted)
1567{
1568 dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con,
1569 msg, aborted, con->v2.out_epil.front_crc,
1570 con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc);
1571
1572 encode_epilogue_plain(con, aborted);
1573 add_out_kvec(con, buf: &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN);
1574}
1575
1576/*
1577 * For "used" empty segments, crc is -1. For unused (trailing)
1578 * segments, crc is 0.
1579 */
1580static void prepare_message_plain(struct ceph_connection *con,
1581 struct ceph_msg *msg)
1582{
1583 prepare_head_plain(con, base: con->v2.out_buf,
1584 ctrl_len: sizeof(struct ceph_msg_header2), NULL, extdata_len: 0, to_be_signed: false);
1585
1586 if (!front_len(msg) && !middle_len(msg)) {
1587 if (!data_len(msg)) {
1588 /*
1589 * Empty message: once the head is written,
1590 * we are done -- there is no epilogue.
1591 */
1592 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1593 return;
1594 }
1595
1596 con->v2.out_epil.front_crc = -1;
1597 con->v2.out_epil.middle_crc = -1;
1598 con->v2.out_state = OUT_S_QUEUE_DATA;
1599 return;
1600 }
1601
1602 if (front_len(msg)) {
1603 con->v2.out_epil.front_crc = crc32c(crc: -1, p: msg->front.iov_base,
1604 len: front_len(msg));
1605 add_out_kvec(con, buf: msg->front.iov_base, len: front_len(msg));
1606 } else {
1607 /* middle (at least) is there, checked above */
1608 con->v2.out_epil.front_crc = -1;
1609 }
1610
1611 if (middle_len(msg)) {
1612 con->v2.out_epil.middle_crc =
1613 crc32c(crc: -1, p: msg->middle->vec.iov_base, len: middle_len(msg));
1614 add_out_kvec(con, buf: msg->middle->vec.iov_base, len: middle_len(msg));
1615 } else {
1616 con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0;
1617 }
1618
1619 if (data_len(msg)) {
1620 con->v2.out_state = OUT_S_QUEUE_DATA;
1621 } else {
1622 con->v2.out_epil.data_crc = 0;
1623 prepare_epilogue_plain(con, msg, aborted: false);
1624 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1625 }
1626}
1627
1628/*
1629 * Unfortunately the kernel crypto API doesn't support streaming
1630 * (piecewise) operation for AEAD algorithms, so we can't get away
1631 * with a fixed size buffer and a couple sgs. Instead, we have to
1632 * allocate pages for the entire tail of the message (currently up
1633 * to ~32M) and two sgs arrays (up to ~256K each)...
1634 */
1635static int prepare_message_secure(struct ceph_connection *con,
1636 struct ceph_msg *msg)
1637{
1638 void *zerop = page_address(ceph_zero_page);
1639 struct sg_table enc_sgt = {};
1640 struct sg_table sgt = {};
1641 struct page **enc_pages;
1642 int enc_page_cnt;
1643 int tail_len;
1644 int ret;
1645
1646 ret = prepare_head_secure_small(con, base: con->v2.out_buf,
1647 ctrl_len: sizeof(struct ceph_msg_header2));
1648 if (ret)
1649 return ret;
1650
1651 tail_len = tail_onwire_len(msg, secure: true);
1652 if (!tail_len) {
1653 /*
1654 * Empty message: once the head is written,
1655 * we are done -- there is no epilogue.
1656 */
1657 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1658 return 0;
1659 }
1660
1661 encode_epilogue_secure(con, aborted: false);
1662 ret = setup_message_sgs(sgt: &sgt, msg, front_pad: zerop, middle_pad: zerop, data_pad: zerop,
1663 epilogue: &con->v2.out_epil, NULL, dpos: 0, add_tag: false);
1664 if (ret)
1665 goto out;
1666
1667 enc_page_cnt = calc_pages_for(off: 0, len: tail_len);
1668 enc_pages = ceph_alloc_page_vector(num_pages: enc_page_cnt, GFP_NOIO);
1669 if (IS_ERR(ptr: enc_pages)) {
1670 ret = PTR_ERR(ptr: enc_pages);
1671 goto out;
1672 }
1673
1674 WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt);
1675 con->v2.out_enc_pages = enc_pages;
1676 con->v2.out_enc_page_cnt = enc_page_cnt;
1677 con->v2.out_enc_resid = tail_len;
1678 con->v2.out_enc_i = 0;
1679
1680 ret = sg_alloc_table_from_pages(sgt: &enc_sgt, pages: enc_pages, n_pages: enc_page_cnt,
1681 offset: 0, size: tail_len, GFP_NOIO);
1682 if (ret)
1683 goto out;
1684
1685 ret = gcm_crypt(con, encrypt: true, src: sgt.sgl, dst: enc_sgt.sgl,
1686 src_len: tail_len - CEPH_GCM_TAG_LEN);
1687 if (ret)
1688 goto out;
1689
1690 dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con,
1691 msg, sgt.orig_nents, enc_page_cnt);
1692 con->v2.out_state = OUT_S_QUEUE_ENC_PAGE;
1693
1694out:
1695 sg_free_table(&sgt);
1696 sg_free_table(&enc_sgt);
1697 return ret;
1698}
1699
1700static int prepare_message(struct ceph_connection *con, struct ceph_msg *msg)
1701{
1702 int lens[] = {
1703 sizeof(struct ceph_msg_header2),
1704 front_len(msg),
1705 middle_len(msg),
1706 data_len(msg)
1707 };
1708 struct ceph_frame_desc desc;
1709 int ret;
1710
1711 dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con,
1712 msg, lens[0], lens[1], lens[2], lens[3]);
1713
1714 if (con->in_seq > con->in_seq_acked) {
1715 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1716 con->in_seq_acked, con->in_seq);
1717 con->in_seq_acked = con->in_seq;
1718 }
1719
1720 reset_out_kvecs(con);
1721 init_frame_desc(desc: &desc, FRAME_TAG_MESSAGE, lens, len_cnt: 4);
1722 encode_preamble(desc: &desc, p: con->v2.out_buf);
1723 fill_header2(CTRL_BODY(con->v2.out_buf), hdr: &msg->hdr,
1724 ack_seq: con->in_seq_acked);
1725
1726 if (con_secure(con)) {
1727 ret = prepare_message_secure(con, msg);
1728 if (ret)
1729 return ret;
1730 } else {
1731 prepare_message_plain(con, msg);
1732 }
1733
1734 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1735 return 0;
1736}
1737
1738static int prepare_read_banner_prefix(struct ceph_connection *con)
1739{
1740 void *buf;
1741
1742 buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN);
1743 if (!buf)
1744 return -ENOMEM;
1745
1746 reset_in_kvecs(con);
1747 add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1748 add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1749 con->state = CEPH_CON_S_V2_BANNER_PREFIX;
1750 return 0;
1751}
1752
1753static int prepare_read_banner_payload(struct ceph_connection *con,
1754 int payload_len)
1755{
1756 void *buf;
1757
1758 buf = alloc_conn_buf(con, len: payload_len);
1759 if (!buf)
1760 return -ENOMEM;
1761
1762 reset_in_kvecs(con);
1763 add_in_kvec(con, buf, len: payload_len);
1764 add_in_sign_kvec(con, buf, len: payload_len);
1765 con->state = CEPH_CON_S_V2_BANNER_PAYLOAD;
1766 return 0;
1767}
1768
1769static void prepare_read_preamble(struct ceph_connection *con)
1770{
1771 reset_in_kvecs(con);
1772 add_in_kvec(con, buf: con->v2.in_buf,
1773 len: con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN :
1774 CEPH_PREAMBLE_PLAIN_LEN);
1775 con->v2.in_state = IN_S_HANDLE_PREAMBLE;
1776}
1777
1778static int prepare_read_control(struct ceph_connection *con)
1779{
1780 int ctrl_len = con->v2.in_desc.fd_lens[0];
1781 int head_len;
1782 void *buf;
1783
1784 reset_in_kvecs(con);
1785 if (con->state == CEPH_CON_S_V2_HELLO ||
1786 con->state == CEPH_CON_S_V2_AUTH) {
1787 head_len = head_onwire_len(ctrl_len, secure: false);
1788 buf = alloc_conn_buf(con, len: head_len);
1789 if (!buf)
1790 return -ENOMEM;
1791
1792 /* preserve preamble */
1793 memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN);
1794
1795 add_in_kvec(con, CTRL_BODY(buf), len: ctrl_len);
1796 add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN);
1797 add_in_sign_kvec(con, buf, len: head_len);
1798 } else {
1799 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
1800 buf = alloc_conn_buf(con, len: ctrl_len);
1801 if (!buf)
1802 return -ENOMEM;
1803
1804 add_in_kvec(con, buf, len: ctrl_len);
1805 } else {
1806 add_in_kvec(con, CTRL_BODY(con->v2.in_buf), len: ctrl_len);
1807 }
1808 add_in_kvec(con, buf: con->v2.in_buf, CEPH_CRC_LEN);
1809 }
1810 con->v2.in_state = IN_S_HANDLE_CONTROL;
1811 return 0;
1812}
1813
1814static int prepare_read_control_remainder(struct ceph_connection *con)
1815{
1816 int ctrl_len = con->v2.in_desc.fd_lens[0];
1817 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1818 void *buf;
1819
1820 buf = alloc_conn_buf(con, len: ctrl_len);
1821 if (!buf)
1822 return -ENOMEM;
1823
1824 memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN);
1825
1826 reset_in_kvecs(con);
1827 add_in_kvec(con, buf: buf + CEPH_PREAMBLE_INLINE_LEN, len: rem_len);
1828 add_in_kvec(con, buf: con->v2.in_buf,
1829 len: padding_len(len: rem_len) + CEPH_GCM_TAG_LEN);
1830 con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER;
1831 return 0;
1832}
1833
1834static int prepare_read_data(struct ceph_connection *con)
1835{
1836 struct bio_vec bv;
1837
1838 con->in_data_crc = -1;
1839 ceph_msg_data_cursor_init(cursor: &con->v2.in_cursor, msg: con->in_msg,
1840 length: data_len(msg: con->in_msg));
1841
1842 get_bvec_at(cursor: &con->v2.in_cursor, bv: &bv);
1843 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1844 if (unlikely(!con->bounce_page)) {
1845 con->bounce_page = alloc_page(GFP_NOIO);
1846 if (!con->bounce_page) {
1847 pr_err("failed to allocate bounce page\n");
1848 return -ENOMEM;
1849 }
1850 }
1851
1852 bv.bv_page = con->bounce_page;
1853 bv.bv_offset = 0;
1854 }
1855 set_in_bvec(con, bv: &bv);
1856 con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT;
1857 return 0;
1858}
1859
1860static void prepare_read_data_cont(struct ceph_connection *con)
1861{
1862 struct bio_vec bv;
1863
1864 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1865 con->in_data_crc = crc32c(crc: con->in_data_crc,
1866 page_address(con->bounce_page),
1867 len: con->v2.in_bvec.bv_len);
1868
1869 get_bvec_at(cursor: &con->v2.in_cursor, bv: &bv);
1870 memcpy_to_page(page: bv.bv_page, offset: bv.bv_offset,
1871 page_address(con->bounce_page),
1872 len: con->v2.in_bvec.bv_len);
1873 } else {
1874 con->in_data_crc = ceph_crc32c_page(crc: con->in_data_crc,
1875 page: con->v2.in_bvec.bv_page,
1876 page_offset: con->v2.in_bvec.bv_offset,
1877 length: con->v2.in_bvec.bv_len);
1878 }
1879
1880 ceph_msg_data_advance(cursor: &con->v2.in_cursor, bytes: con->v2.in_bvec.bv_len);
1881 if (con->v2.in_cursor.total_resid) {
1882 get_bvec_at(cursor: &con->v2.in_cursor, bv: &bv);
1883 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1884 bv.bv_page = con->bounce_page;
1885 bv.bv_offset = 0;
1886 }
1887 set_in_bvec(con, bv: &bv);
1888 WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT);
1889 return;
1890 }
1891
1892 /*
1893 * We've read all data. Prepare to read epilogue.
1894 */
1895 reset_in_kvecs(con);
1896 add_in_kvec(con, buf: con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1897 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1898}
1899
1900static int prepare_sparse_read_cont(struct ceph_connection *con)
1901{
1902 int ret;
1903 struct bio_vec bv;
1904 char *buf = NULL;
1905 struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;
1906
1907 WARN_ON(con->v2.in_state != IN_S_PREPARE_SPARSE_DATA_CONT);
1908
1909 if (iov_iter_is_bvec(i: &con->v2.in_iter)) {
1910 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1911 con->in_data_crc = crc32c(crc: con->in_data_crc,
1912 page_address(con->bounce_page),
1913 len: con->v2.in_bvec.bv_len);
1914 get_bvec_at(cursor, bv: &bv);
1915 memcpy_to_page(page: bv.bv_page, offset: bv.bv_offset,
1916 page_address(con->bounce_page),
1917 len: con->v2.in_bvec.bv_len);
1918 } else {
1919 con->in_data_crc = ceph_crc32c_page(crc: con->in_data_crc,
1920 page: con->v2.in_bvec.bv_page,
1921 page_offset: con->v2.in_bvec.bv_offset,
1922 length: con->v2.in_bvec.bv_len);
1923 }
1924
1925 ceph_msg_data_advance(cursor, bytes: con->v2.in_bvec.bv_len);
1926 cursor->sr_resid -= con->v2.in_bvec.bv_len;
1927 dout("%s: advance by 0x%x sr_resid 0x%x\n", __func__,
1928 con->v2.in_bvec.bv_len, cursor->sr_resid);
1929 WARN_ON_ONCE(cursor->sr_resid > cursor->total_resid);
1930 if (cursor->sr_resid) {
1931 get_bvec_at(cursor, bv: &bv);
1932 if (bv.bv_len > cursor->sr_resid)
1933 bv.bv_len = cursor->sr_resid;
1934 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1935 bv.bv_page = con->bounce_page;
1936 bv.bv_offset = 0;
1937 }
1938 set_in_bvec(con, bv: &bv);
1939 con->v2.data_len_remain -= bv.bv_len;
1940 return 0;
1941 }
1942 } else if (iov_iter_is_kvec(i: &con->v2.in_iter)) {
1943 /* On first call, we have no kvec so don't compute crc */
1944 if (con->v2.in_kvec_cnt) {
1945 WARN_ON_ONCE(con->v2.in_kvec_cnt > 1);
1946 con->in_data_crc = crc32c(crc: con->in_data_crc,
1947 p: con->v2.in_kvecs[0].iov_base,
1948 len: con->v2.in_kvecs[0].iov_len);
1949 }
1950 } else {
1951 return -EIO;
1952 }
1953
1954 /* get next extent */
1955 ret = con->ops->sparse_read(con, cursor, &buf);
1956 if (ret <= 0) {
1957 if (ret < 0)
1958 return ret;
1959
1960 reset_in_kvecs(con);
1961 add_in_kvec(con, buf: con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1962 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1963 return 0;
1964 }
1965
1966 if (buf) {
1967 /* receive into buffer */
1968 reset_in_kvecs(con);
1969 add_in_kvec(con, buf, len: ret);
1970 con->v2.data_len_remain -= ret;
1971 return 0;
1972 }
1973
1974 if (ret > cursor->total_resid) {
1975 pr_warn("%s: ret 0x%x total_resid 0x%zx resid 0x%zx\n",
1976 __func__, ret, cursor->total_resid, cursor->resid);
1977 return -EIO;
1978 }
1979 get_bvec_at(cursor, bv: &bv);
1980 if (bv.bv_len > cursor->sr_resid)
1981 bv.bv_len = cursor->sr_resid;
1982 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1983 if (unlikely(!con->bounce_page)) {
1984 con->bounce_page = alloc_page(GFP_NOIO);
1985 if (!con->bounce_page) {
1986 pr_err("failed to allocate bounce page\n");
1987 return -ENOMEM;
1988 }
1989 }
1990
1991 bv.bv_page = con->bounce_page;
1992 bv.bv_offset = 0;
1993 }
1994 set_in_bvec(con, bv: &bv);
1995 con->v2.data_len_remain -= ret;
1996 return ret;
1997}
1998
1999static int prepare_sparse_read_data(struct ceph_connection *con)
2000{
2001 struct ceph_msg *msg = con->in_msg;
2002
2003 dout("%s: starting sparse read\n", __func__);
2004
2005 if (WARN_ON_ONCE(!con->ops->sparse_read))
2006 return -EOPNOTSUPP;
2007
2008 if (!con_secure(con))
2009 con->in_data_crc = -1;
2010
2011 ceph_msg_data_cursor_init(cursor: &con->v2.in_cursor, msg,
2012 length: msg->sparse_read_total);
2013
2014 reset_in_kvecs(con);
2015 con->v2.in_state = IN_S_PREPARE_SPARSE_DATA_CONT;
2016 con->v2.data_len_remain = data_len(msg);
2017 return prepare_sparse_read_cont(con);
2018}
2019
2020static int prepare_read_tail_plain(struct ceph_connection *con)
2021{
2022 struct ceph_msg *msg = con->in_msg;
2023
2024 if (!front_len(msg) && !middle_len(msg)) {
2025 WARN_ON(!data_len(msg));
2026 return prepare_read_data(con);
2027 }
2028
2029 reset_in_kvecs(con);
2030 if (front_len(msg)) {
2031 add_in_kvec(con, buf: msg->front.iov_base, len: front_len(msg));
2032 WARN_ON(msg->front.iov_len != front_len(msg));
2033 }
2034 if (middle_len(msg)) {
2035 add_in_kvec(con, buf: msg->middle->vec.iov_base, len: middle_len(msg));
2036 WARN_ON(msg->middle->vec.iov_len != middle_len(msg));
2037 }
2038
2039 if (data_len(msg)) {
2040 if (msg->sparse_read_total)
2041 con->v2.in_state = IN_S_PREPARE_SPARSE_DATA;
2042 else
2043 con->v2.in_state = IN_S_PREPARE_READ_DATA;
2044 } else {
2045 add_in_kvec(con, buf: con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
2046 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
2047 }
2048 return 0;
2049}
2050
2051static void prepare_read_enc_page(struct ceph_connection *con)
2052{
2053 struct bio_vec bv;
2054
2055 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i,
2056 con->v2.in_enc_resid);
2057 WARN_ON(!con->v2.in_enc_resid);
2058
2059 bvec_set_page(bv: &bv, page: con->v2.in_enc_pages[con->v2.in_enc_i],
2060 min(con->v2.in_enc_resid, (int)PAGE_SIZE), offset: 0);
2061
2062 set_in_bvec(con, bv: &bv);
2063 con->v2.in_enc_i++;
2064 con->v2.in_enc_resid -= bv.bv_len;
2065
2066 if (con->v2.in_enc_resid) {
2067 con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE;
2068 return;
2069 }
2070
2071 /*
2072 * We are set to read the last piece of ciphertext (ending
2073 * with epilogue) + auth tag.
2074 */
2075 WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt);
2076 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
2077}
2078
2079static int prepare_read_tail_secure(struct ceph_connection *con)
2080{
2081 struct page **enc_pages;
2082 int enc_page_cnt;
2083 int tail_len;
2084
2085 tail_len = tail_onwire_len(msg: con->in_msg, secure: true);
2086 WARN_ON(!tail_len);
2087
2088 enc_page_cnt = calc_pages_for(off: 0, len: tail_len);
2089 enc_pages = ceph_alloc_page_vector(num_pages: enc_page_cnt, GFP_NOIO);
2090 if (IS_ERR(ptr: enc_pages))
2091 return PTR_ERR(ptr: enc_pages);
2092
2093 WARN_ON(con->v2.in_enc_pages || con->v2.in_enc_page_cnt);
2094 con->v2.in_enc_pages = enc_pages;
2095 con->v2.in_enc_page_cnt = enc_page_cnt;
2096 con->v2.in_enc_resid = tail_len;
2097 con->v2.in_enc_i = 0;
2098
2099 prepare_read_enc_page(con);
2100 return 0;
2101}
2102
2103static void __finish_skip(struct ceph_connection *con)
2104{
2105 con->in_seq++;
2106 prepare_read_preamble(con);
2107}
2108
2109static void prepare_skip_message(struct ceph_connection *con)
2110{
2111 struct ceph_frame_desc *desc = &con->v2.in_desc;
2112 int tail_len;
2113
2114 dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1],
2115 desc->fd_lens[2], desc->fd_lens[3]);
2116
2117 tail_len = __tail_onwire_len(front_len: desc->fd_lens[1], middle_len: desc->fd_lens[2],
2118 data_len: desc->fd_lens[3], secure: con_secure(con));
2119 if (!tail_len) {
2120 __finish_skip(con);
2121 } else {
2122 set_in_skip(con, len: tail_len);
2123 con->v2.in_state = IN_S_FINISH_SKIP;
2124 }
2125}
2126
2127static int process_banner_prefix(struct ceph_connection *con)
2128{
2129 int payload_len;
2130 void *p;
2131
2132 WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN);
2133
2134 p = con->v2.in_kvecs[0].iov_base;
2135 if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) {
2136 if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN))
2137 con->error_msg = "server is speaking msgr1 protocol";
2138 else
2139 con->error_msg = "protocol error, bad banner";
2140 return -EINVAL;
2141 }
2142
2143 p += CEPH_BANNER_V2_LEN;
2144 payload_len = ceph_decode_16(p: &p);
2145 dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2146
2147 return prepare_read_banner_payload(con, payload_len);
2148}
2149
2150static int process_banner_payload(struct ceph_connection *con)
2151{
2152 void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len;
2153 u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES;
2154 u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES;
2155 u64 server_feat, server_req_feat;
2156 void *p;
2157 int ret;
2158
2159 p = con->v2.in_kvecs[0].iov_base;
2160 ceph_decode_64_safe(&p, end, server_feat, bad);
2161 ceph_decode_64_safe(&p, end, server_req_feat, bad);
2162
2163 dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n",
2164 __func__, con, server_feat, server_req_feat);
2165
2166 if (req_feat & ~server_feat) {
2167 pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2168 server_feat, req_feat & ~server_feat);
2169 con->error_msg = "missing required protocol features";
2170 return -EINVAL;
2171 }
2172 if (server_req_feat & ~feat) {
2173 pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2174 feat, server_req_feat & ~feat);
2175 con->error_msg = "missing required protocol features";
2176 return -EINVAL;
2177 }
2178
2179 /* no reset_out_kvecs() as our banner may still be pending */
2180 ret = prepare_hello(con);
2181 if (ret) {
2182 pr_err("prepare_hello failed: %d\n", ret);
2183 return ret;
2184 }
2185
2186 con->state = CEPH_CON_S_V2_HELLO;
2187 prepare_read_preamble(con);
2188 return 0;
2189
2190bad:
2191 pr_err("failed to decode banner payload\n");
2192 return -EINVAL;
2193}
2194
2195static int process_hello(struct ceph_connection *con, void *p, void *end)
2196{
2197 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
2198 struct ceph_entity_addr addr_for_me;
2199 u8 entity_type;
2200 int ret;
2201
2202 if (con->state != CEPH_CON_S_V2_HELLO) {
2203 con->error_msg = "protocol error, unexpected hello";
2204 return -EINVAL;
2205 }
2206
2207 ceph_decode_8_safe(&p, end, entity_type, bad);
2208 ret = ceph_decode_entity_addr(p: &p, end, addr: &addr_for_me);
2209 if (ret) {
2210 pr_err("failed to decode addr_for_me: %d\n", ret);
2211 return ret;
2212 }
2213
2214 dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con,
2215 entity_type, ceph_pr_addr(&addr_for_me));
2216
2217 if (entity_type != con->peer_name.type) {
2218 pr_err("bad peer type, want %d, got %d\n",
2219 con->peer_name.type, entity_type);
2220 con->error_msg = "wrong peer at address";
2221 return -EINVAL;
2222 }
2223
2224 /*
2225 * Set our address to the address our first peer (i.e. monitor)
2226 * sees that we are connecting from. If we are behind some sort
2227 * of NAT and want to be identified by some private (not NATed)
2228 * address, ip option should be used.
2229 */
2230 if (ceph_addr_is_blank(addr: my_addr)) {
2231 memcpy(&my_addr->in_addr, &addr_for_me.in_addr,
2232 sizeof(my_addr->in_addr));
2233 ceph_addr_set_port(addr: my_addr, p: 0);
2234 dout("%s con %p set my addr %s, as seen by peer %s\n",
2235 __func__, con, ceph_pr_addr(my_addr),
2236 ceph_pr_addr(&con->peer_addr));
2237 } else {
2238 dout("%s con %p my addr already set %s\n",
2239 __func__, con, ceph_pr_addr(my_addr));
2240 }
2241
2242 WARN_ON(ceph_addr_is_blank(my_addr) || ceph_addr_port(my_addr));
2243 WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY);
2244 WARN_ON(!my_addr->nonce);
2245
2246 /* no reset_out_kvecs() as our hello may still be pending */
2247 ret = prepare_auth_request(con);
2248 if (ret) {
2249 if (ret != -EAGAIN)
2250 pr_err("prepare_auth_request failed: %d\n", ret);
2251 return ret;
2252 }
2253
2254 con->state = CEPH_CON_S_V2_AUTH;
2255 return 0;
2256
2257bad:
2258 pr_err("failed to decode hello\n");
2259 return -EINVAL;
2260}
2261
2262static int process_auth_bad_method(struct ceph_connection *con,
2263 void *p, void *end)
2264{
2265 int allowed_protos[8], allowed_modes[8];
2266 int allowed_proto_cnt, allowed_mode_cnt;
2267 int used_proto, result;
2268 int ret;
2269 int i;
2270
2271 if (con->state != CEPH_CON_S_V2_AUTH) {
2272 con->error_msg = "protocol error, unexpected auth_bad_method";
2273 return -EINVAL;
2274 }
2275
2276 ceph_decode_32_safe(&p, end, used_proto, bad);
2277 ceph_decode_32_safe(&p, end, result, bad);
2278 dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto,
2279 result);
2280
2281 ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad);
2282 if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) {
2283 pr_err("allowed_protos too big %d\n", allowed_proto_cnt);
2284 return -EINVAL;
2285 }
2286 for (i = 0; i < allowed_proto_cnt; i++) {
2287 ceph_decode_32_safe(&p, end, allowed_protos[i], bad);
2288 dout("%s con %p allowed_protos[%d] %d\n", __func__, con,
2289 i, allowed_protos[i]);
2290 }
2291
2292 ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad);
2293 if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) {
2294 pr_err("allowed_modes too big %d\n", allowed_mode_cnt);
2295 return -EINVAL;
2296 }
2297 for (i = 0; i < allowed_mode_cnt; i++) {
2298 ceph_decode_32_safe(&p, end, allowed_modes[i], bad);
2299 dout("%s con %p allowed_modes[%d] %d\n", __func__, con,
2300 i, allowed_modes[i]);
2301 }
2302
2303 mutex_unlock(lock: &con->mutex);
2304 ret = con->ops->handle_auth_bad_method(con, used_proto, result,
2305 allowed_protos,
2306 allowed_proto_cnt,
2307 allowed_modes,
2308 allowed_mode_cnt);
2309 mutex_lock(&con->mutex);
2310 if (con->state != CEPH_CON_S_V2_AUTH) {
2311 dout("%s con %p state changed to %d\n", __func__, con,
2312 con->state);
2313 return -EAGAIN;
2314 }
2315
2316 dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret);
2317 return ret;
2318
2319bad:
2320 pr_err("failed to decode auth_bad_method\n");
2321 return -EINVAL;
2322}
2323
2324static int process_auth_reply_more(struct ceph_connection *con,
2325 void *p, void *end)
2326{
2327 int payload_len;
2328 int ret;
2329
2330 if (con->state != CEPH_CON_S_V2_AUTH) {
2331 con->error_msg = "protocol error, unexpected auth_reply_more";
2332 return -EINVAL;
2333 }
2334
2335 ceph_decode_32_safe(&p, end, payload_len, bad);
2336 ceph_decode_need(&p, end, payload_len, bad);
2337
2338 dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2339
2340 reset_out_kvecs(con);
2341 ret = prepare_auth_request_more(con, reply: p, reply_len: payload_len);
2342 if (ret) {
2343 if (ret != -EAGAIN)
2344 pr_err("prepare_auth_request_more failed: %d\n", ret);
2345 return ret;
2346 }
2347
2348 return 0;
2349
2350bad:
2351 pr_err("failed to decode auth_reply_more\n");
2352 return -EINVAL;
2353}
2354
2355/*
2356 * Align session_key and con_secret to avoid GFP_ATOMIC allocation
2357 * inside crypto_shash_setkey() and crypto_aead_setkey() called from
2358 * setup_crypto(). __aligned(16) isn't guaranteed to work for stack
2359 * objects, so do it by hand.
2360 */
2361static int process_auth_done(struct ceph_connection *con, void *p, void *end)
2362{
2363 u8 session_key_buf[CEPH_KEY_LEN + 16];
2364 u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16];
2365 u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16);
2366 u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16);
2367 int session_key_len, con_secret_len;
2368 int payload_len;
2369 u64 global_id;
2370 int ret;
2371
2372 if (con->state != CEPH_CON_S_V2_AUTH) {
2373 con->error_msg = "protocol error, unexpected auth_done";
2374 return -EINVAL;
2375 }
2376
2377 ceph_decode_64_safe(&p, end, global_id, bad);
2378 ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);
2379
2380 ceph_decode_32_safe(&p, end, payload_len, bad);
2381 ceph_decode_need(&p, end, payload_len, bad);
2382
2383 dout("%s con %p global_id %llu con_mode %d payload_len %d\n",
2384 __func__, con, global_id, con->v2.con_mode, payload_len);
2385
2386 mutex_unlock(lock: &con->mutex);
2387 session_key_len = 0;
2388 con_secret_len = 0;
2389 ret = con->ops->handle_auth_done(con, global_id, p, payload_len,
2390 session_key, &session_key_len,
2391 con_secret, &con_secret_len);
2392 mutex_lock(&con->mutex);
2393 if (con->state != CEPH_CON_S_V2_AUTH) {
2394 dout("%s con %p state changed to %d\n", __func__, con,
2395 con->state);
2396 ret = -EAGAIN;
2397 goto out;
2398 }
2399
2400 dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret);
2401 if (ret)
2402 goto out;
2403
2404 ret = setup_crypto(con, session_key, session_key_len, con_secret,
2405 con_secret_len);
2406 if (ret)
2407 goto out;
2408
2409 reset_out_kvecs(con);
2410 ret = prepare_auth_signature(con);
2411 if (ret) {
2412 pr_err("prepare_auth_signature failed: %d\n", ret);
2413 goto out;
2414 }
2415
2416 con->state = CEPH_CON_S_V2_AUTH_SIGNATURE;
2417
2418out:
2419 memzero_explicit(s: session_key_buf, count: sizeof(session_key_buf));
2420 memzero_explicit(s: con_secret_buf, count: sizeof(con_secret_buf));
2421 return ret;
2422
2423bad:
2424 pr_err("failed to decode auth_done\n");
2425 return -EINVAL;
2426}
2427
2428static int process_auth_signature(struct ceph_connection *con,
2429 void *p, void *end)
2430{
2431 u8 hmac[SHA256_DIGEST_SIZE];
2432 int ret;
2433
2434 if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) {
2435 con->error_msg = "protocol error, unexpected auth_signature";
2436 return -EINVAL;
2437 }
2438
2439 ceph_hmac_sha256(con, kvecs: con->v2.out_sign_kvecs, kvec_cnt: con->v2.out_sign_kvec_cnt,
2440 hmac);
2441
2442 ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad);
2443 if (crypto_memneq(a: p, b: hmac, SHA256_DIGEST_SIZE)) {
2444 con->error_msg = "integrity error, bad auth signature";
2445 return -EBADMSG;
2446 }
2447
2448 dout("%s con %p auth signature ok\n", __func__, con);
2449
2450 /* no reset_out_kvecs() as our auth_signature may still be pending */
2451 if (!con->v2.server_cookie) {
2452 ret = prepare_client_ident(con);
2453 if (ret) {
2454 pr_err("prepare_client_ident failed: %d\n", ret);
2455 return ret;
2456 }
2457
2458 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2459 } else {
2460 ret = prepare_session_reconnect(con);
2461 if (ret) {
2462 pr_err("prepare_session_reconnect failed: %d\n", ret);
2463 return ret;
2464 }
2465
2466 con->state = CEPH_CON_S_V2_SESSION_RECONNECT;
2467 }
2468
2469 return 0;
2470
2471bad:
2472 pr_err("failed to decode auth_signature\n");
2473 return -EINVAL;
2474}
2475
2476static int process_server_ident(struct ceph_connection *con,
2477 void *p, void *end)
2478{
2479 struct ceph_client *client = from_msgr(con->msgr);
2480 u64 features, required_features;
2481 struct ceph_entity_addr addr;
2482 u64 global_seq;
2483 u64 global_id;
2484 u64 cookie;
2485 u64 flags;
2486 int ret;
2487
2488 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2489 con->error_msg = "protocol error, unexpected server_ident";
2490 return -EINVAL;
2491 }
2492
2493 ret = ceph_decode_entity_addrvec(p: &p, end, msgr2: true, addr: &addr);
2494 if (ret) {
2495 pr_err("failed to decode server addrs: %d\n", ret);
2496 return ret;
2497 }
2498
2499 ceph_decode_64_safe(&p, end, global_id, bad);
2500 ceph_decode_64_safe(&p, end, global_seq, bad);
2501 ceph_decode_64_safe(&p, end, features, bad);
2502 ceph_decode_64_safe(&p, end, required_features, bad);
2503 ceph_decode_64_safe(&p, end, flags, bad);
2504 ceph_decode_64_safe(&p, end, cookie, bad);
2505
2506 dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n",
2507 __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce),
2508 global_id, global_seq, features, required_features, flags, cookie);
2509
2510 /* is this who we intended to talk to? */
2511 if (memcmp(p: &addr, q: &con->peer_addr, size: sizeof(con->peer_addr))) {
2512 pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n",
2513 ceph_pr_addr(&con->peer_addr),
2514 le32_to_cpu(con->peer_addr.nonce),
2515 ceph_pr_addr(&addr), le32_to_cpu(addr.nonce));
2516 con->error_msg = "wrong peer at address";
2517 return -EINVAL;
2518 }
2519
2520 if (client->required_features & ~features) {
2521 pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2522 features, client->required_features & ~features);
2523 con->error_msg = "missing required protocol features";
2524 return -EINVAL;
2525 }
2526
2527 /*
2528 * Both name->type and name->num are set in ceph_con_open() but
2529 * name->num may be bogus in the initial monmap. name->type is
2530 * verified in handle_hello().
2531 */
2532 WARN_ON(!con->peer_name.type);
2533 con->peer_name.num = cpu_to_le64(global_id);
2534 con->v2.peer_global_seq = global_seq;
2535 con->peer_features = features;
2536 WARN_ON(required_features & ~client->supported_features);
2537 con->v2.server_cookie = cookie;
2538
2539 if (flags & CEPH_MSG_CONNECT_LOSSY) {
2540 ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX);
2541 WARN_ON(con->v2.server_cookie);
2542 } else {
2543 WARN_ON(!con->v2.server_cookie);
2544 }
2545
2546 clear_in_sign_kvecs(con);
2547 clear_out_sign_kvecs(con);
2548 free_conn_bufs(con);
2549 con->delay = 0; /* reset backoff memory */
2550
2551 con->state = CEPH_CON_S_OPEN;
2552 con->v2.out_state = OUT_S_GET_NEXT;
2553 return 0;
2554
2555bad:
2556 pr_err("failed to decode server_ident\n");
2557 return -EINVAL;
2558}
2559
2560static int process_ident_missing_features(struct ceph_connection *con,
2561 void *p, void *end)
2562{
2563 struct ceph_client *client = from_msgr(con->msgr);
2564 u64 missing_features;
2565
2566 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2567 con->error_msg = "protocol error, unexpected ident_missing_features";
2568 return -EINVAL;
2569 }
2570
2571 ceph_decode_64_safe(&p, end, missing_features, bad);
2572 pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2573 client->supported_features, missing_features);
2574 con->error_msg = "missing required protocol features";
2575 return -EINVAL;
2576
2577bad:
2578 pr_err("failed to decode ident_missing_features\n");
2579 return -EINVAL;
2580}
2581
2582static int process_session_reconnect_ok(struct ceph_connection *con,
2583 void *p, void *end)
2584{
2585 u64 seq;
2586
2587 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2588 con->error_msg = "protocol error, unexpected session_reconnect_ok";
2589 return -EINVAL;
2590 }
2591
2592 ceph_decode_64_safe(&p, end, seq, bad);
2593
2594 dout("%s con %p seq %llu\n", __func__, con, seq);
2595 ceph_con_discard_requeued(con, reconnect_seq: seq);
2596
2597 clear_in_sign_kvecs(con);
2598 clear_out_sign_kvecs(con);
2599 free_conn_bufs(con);
2600 con->delay = 0; /* reset backoff memory */
2601
2602 con->state = CEPH_CON_S_OPEN;
2603 con->v2.out_state = OUT_S_GET_NEXT;
2604 return 0;
2605
2606bad:
2607 pr_err("failed to decode session_reconnect_ok\n");
2608 return -EINVAL;
2609}
2610
2611static int process_session_retry(struct ceph_connection *con,
2612 void *p, void *end)
2613{
2614 u64 connect_seq;
2615 int ret;
2616
2617 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2618 con->error_msg = "protocol error, unexpected session_retry";
2619 return -EINVAL;
2620 }
2621
2622 ceph_decode_64_safe(&p, end, connect_seq, bad);
2623
2624 dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq);
2625 WARN_ON(connect_seq <= con->v2.connect_seq);
2626 con->v2.connect_seq = connect_seq + 1;
2627
2628 free_conn_bufs(con);
2629
2630 reset_out_kvecs(con);
2631 ret = prepare_session_reconnect(con);
2632 if (ret) {
2633 pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret);
2634 return ret;
2635 }
2636
2637 return 0;
2638
2639bad:
2640 pr_err("failed to decode session_retry\n");
2641 return -EINVAL;
2642}
2643
2644static int process_session_retry_global(struct ceph_connection *con,
2645 void *p, void *end)
2646{
2647 u64 global_seq;
2648 int ret;
2649
2650 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2651 con->error_msg = "protocol error, unexpected session_retry_global";
2652 return -EINVAL;
2653 }
2654
2655 ceph_decode_64_safe(&p, end, global_seq, bad);
2656
2657 dout("%s con %p global_seq %llu\n", __func__, con, global_seq);
2658 WARN_ON(global_seq <= con->v2.global_seq);
2659 con->v2.global_seq = ceph_get_global_seq(msgr: con->msgr, gt: global_seq);
2660
2661 free_conn_bufs(con);
2662
2663 reset_out_kvecs(con);
2664 ret = prepare_session_reconnect(con);
2665 if (ret) {
2666 pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret);
2667 return ret;
2668 }
2669
2670 return 0;
2671
2672bad:
2673 pr_err("failed to decode session_retry_global\n");
2674 return -EINVAL;
2675}
2676
2677static int process_session_reset(struct ceph_connection *con,
2678 void *p, void *end)
2679{
2680 bool full;
2681 int ret;
2682
2683 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2684 con->error_msg = "protocol error, unexpected session_reset";
2685 return -EINVAL;
2686 }
2687
2688 ceph_decode_8_safe(&p, end, full, bad);
2689 if (!full) {
2690 con->error_msg = "protocol error, bad session_reset";
2691 return -EINVAL;
2692 }
2693
2694 pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name),
2695 ceph_pr_addr(&con->peer_addr));
2696 ceph_con_reset_session(con);
2697
2698 mutex_unlock(lock: &con->mutex);
2699 if (con->ops->peer_reset)
2700 con->ops->peer_reset(con);
2701 mutex_lock(&con->mutex);
2702 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2703 dout("%s con %p state changed to %d\n", __func__, con,
2704 con->state);
2705 return -EAGAIN;
2706 }
2707
2708 free_conn_bufs(con);
2709
2710 reset_out_kvecs(con);
2711 ret = prepare_client_ident(con);
2712 if (ret) {
2713 pr_err("prepare_client_ident (rst) failed: %d\n", ret);
2714 return ret;
2715 }
2716
2717 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2718 return 0;
2719
2720bad:
2721 pr_err("failed to decode session_reset\n");
2722 return -EINVAL;
2723}
2724
2725static int process_keepalive2_ack(struct ceph_connection *con,
2726 void *p, void *end)
2727{
2728 if (con->state != CEPH_CON_S_OPEN) {
2729 con->error_msg = "protocol error, unexpected keepalive2_ack";
2730 return -EINVAL;
2731 }
2732
2733 ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad);
2734 ceph_decode_timespec64(ts: &con->last_keepalive_ack, tv: p);
2735
2736 dout("%s con %p timestamp %ptSp\n", __func__, con, &con->last_keepalive_ack);
2737
2738 return 0;
2739
2740bad:
2741 pr_err("failed to decode keepalive2_ack\n");
2742 return -EINVAL;
2743}
2744
2745static int process_ack(struct ceph_connection *con, void *p, void *end)
2746{
2747 u64 seq;
2748
2749 if (con->state != CEPH_CON_S_OPEN) {
2750 con->error_msg = "protocol error, unexpected ack";
2751 return -EINVAL;
2752 }
2753
2754 ceph_decode_64_safe(&p, end, seq, bad);
2755
2756 dout("%s con %p seq %llu\n", __func__, con, seq);
2757 ceph_con_discard_sent(con, ack_seq: seq);
2758 return 0;
2759
2760bad:
2761 pr_err("failed to decode ack\n");
2762 return -EINVAL;
2763}
2764
2765static int process_control(struct ceph_connection *con, void *p, void *end)
2766{
2767 int tag = con->v2.in_desc.fd_tag;
2768 int ret;
2769
2770 dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p));
2771
2772 switch (tag) {
2773 case FRAME_TAG_HELLO:
2774 ret = process_hello(con, p, end);
2775 break;
2776 case FRAME_TAG_AUTH_BAD_METHOD:
2777 ret = process_auth_bad_method(con, p, end);
2778 break;
2779 case FRAME_TAG_AUTH_REPLY_MORE:
2780 ret = process_auth_reply_more(con, p, end);
2781 break;
2782 case FRAME_TAG_AUTH_DONE:
2783 ret = process_auth_done(con, p, end);
2784 break;
2785 case FRAME_TAG_AUTH_SIGNATURE:
2786 ret = process_auth_signature(con, p, end);
2787 break;
2788 case FRAME_TAG_SERVER_IDENT:
2789 ret = process_server_ident(con, p, end);
2790 break;
2791 case FRAME_TAG_IDENT_MISSING_FEATURES:
2792 ret = process_ident_missing_features(con, p, end);
2793 break;
2794 case FRAME_TAG_SESSION_RECONNECT_OK:
2795 ret = process_session_reconnect_ok(con, p, end);
2796 break;
2797 case FRAME_TAG_SESSION_RETRY:
2798 ret = process_session_retry(con, p, end);
2799 break;
2800 case FRAME_TAG_SESSION_RETRY_GLOBAL:
2801 ret = process_session_retry_global(con, p, end);
2802 break;
2803 case FRAME_TAG_SESSION_RESET:
2804 ret = process_session_reset(con, p, end);
2805 break;
2806 case FRAME_TAG_KEEPALIVE2_ACK:
2807 ret = process_keepalive2_ack(con, p, end);
2808 break;
2809 case FRAME_TAG_ACK:
2810 ret = process_ack(con, p, end);
2811 break;
2812 default:
2813 pr_err("bad tag %d\n", tag);
2814 con->error_msg = "protocol error, bad tag";
2815 return -EINVAL;
2816 }
2817 if (ret) {
2818 dout("%s con %p error %d\n", __func__, con, ret);
2819 return ret;
2820 }
2821
2822 prepare_read_preamble(con);
2823 return 0;
2824}
2825
2826/*
2827 * Return:
2828 * 1 - con->in_msg set, read message
2829 * 0 - skip message
2830 * <0 - error
2831 */
2832static int process_message_header(struct ceph_connection *con,
2833 void *p, void *end)
2834{
2835 struct ceph_frame_desc *desc = &con->v2.in_desc;
2836 struct ceph_msg_header2 *hdr2 = p;
2837 struct ceph_msg_header hdr;
2838 int skip;
2839 int ret;
2840 u64 seq;
2841
2842 /* verify seq# */
2843 seq = le64_to_cpu(hdr2->seq);
2844 if ((s64)seq - (s64)con->in_seq < 1) {
2845 pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n",
2846 ENTITY_NAME(con->peer_name),
2847 ceph_pr_addr(&con->peer_addr),
2848 seq, con->in_seq + 1);
2849 return 0;
2850 }
2851 if ((s64)seq - (s64)con->in_seq > 1) {
2852 pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1);
2853 con->error_msg = "bad message sequence # for incoming message";
2854 return -EBADE;
2855 }
2856
2857 ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq));
2858
2859 fill_header(hdr: &hdr, hdr2, front_len: desc->fd_lens[1], middle_len: desc->fd_lens[2],
2860 data_len: desc->fd_lens[3], peer_name: &con->peer_name);
2861 ret = ceph_con_in_msg_alloc(con, hdr: &hdr, skip: &skip);
2862 if (ret)
2863 return ret;
2864
2865 WARN_ON(!con->in_msg ^ skip);
2866 if (skip)
2867 return 0;
2868
2869 WARN_ON(!con->in_msg);
2870 WARN_ON(con->in_msg->con != con);
2871 return 1;
2872}
2873
2874static int process_message(struct ceph_connection *con)
2875{
2876 ceph_con_process_message(con);
2877
2878 /*
2879 * We could have been closed by ceph_con_close() because
2880 * ceph_con_process_message() temporarily drops con->mutex.
2881 */
2882 if (con->state != CEPH_CON_S_OPEN) {
2883 dout("%s con %p state changed to %d\n", __func__, con,
2884 con->state);
2885 return -EAGAIN;
2886 }
2887
2888 prepare_read_preamble(con);
2889 return 0;
2890}
2891
2892static int __handle_control(struct ceph_connection *con, void *p)
2893{
2894 void *end = p + con->v2.in_desc.fd_lens[0];
2895 struct ceph_msg *msg;
2896 int ret;
2897
2898 if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE)
2899 return process_control(con, p, end);
2900
2901 ret = process_message_header(con, p, end);
2902 if (ret < 0)
2903 return ret;
2904 if (ret == 0) {
2905 prepare_skip_message(con);
2906 return 0;
2907 }
2908
2909 msg = con->in_msg; /* set in process_message_header() */
2910 if (front_len(msg)) {
2911 WARN_ON(front_len(msg) > msg->front_alloc_len);
2912 msg->front.iov_len = front_len(msg);
2913 } else {
2914 msg->front.iov_len = 0;
2915 }
2916 if (middle_len(msg)) {
2917 WARN_ON(middle_len(msg) > msg->middle->alloc_len);
2918 msg->middle->vec.iov_len = middle_len(msg);
2919 } else if (msg->middle) {
2920 msg->middle->vec.iov_len = 0;
2921 }
2922
2923 if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
2924 return process_message(con);
2925
2926 if (con_secure(con))
2927 return prepare_read_tail_secure(con);
2928
2929 return prepare_read_tail_plain(con);
2930}
2931
2932static int handle_preamble(struct ceph_connection *con)
2933{
2934 struct ceph_frame_desc *desc = &con->v2.in_desc;
2935 int ret;
2936
2937 if (con_secure(con)) {
2938 ret = decrypt_preamble(con);
2939 if (ret) {
2940 if (ret == -EBADMSG)
2941 con->error_msg = "integrity error, bad preamble auth tag";
2942 return ret;
2943 }
2944 }
2945
2946 ret = decode_preamble(p: con->v2.in_buf, desc);
2947 if (ret) {
2948 if (ret == -EBADMSG)
2949 con->error_msg = "integrity error, bad crc";
2950 else
2951 con->error_msg = "protocol error, bad preamble";
2952 return ret;
2953 }
2954
2955 dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__,
2956 con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0],
2957 desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]);
2958
2959 if (!con_secure(con))
2960 return prepare_read_control(con);
2961
2962 if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN)
2963 return prepare_read_control_remainder(con);
2964
2965 return __handle_control(con, CTRL_BODY(con->v2.in_buf));
2966}
2967
2968static int handle_control(struct ceph_connection *con)
2969{
2970 int ctrl_len = con->v2.in_desc.fd_lens[0];
2971 void *buf;
2972 int ret;
2973
2974 WARN_ON(con_secure(con));
2975
2976 ret = verify_control_crc(con);
2977 if (ret) {
2978 con->error_msg = "integrity error, bad crc";
2979 return ret;
2980 }
2981
2982 if (con->state == CEPH_CON_S_V2_AUTH) {
2983 buf = alloc_conn_buf(con, len: ctrl_len);
2984 if (!buf)
2985 return -ENOMEM;
2986
2987 memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len);
2988 return __handle_control(con, p: buf);
2989 }
2990
2991 return __handle_control(con, p: con->v2.in_kvecs[0].iov_base);
2992}
2993
2994static int handle_control_remainder(struct ceph_connection *con)
2995{
2996 int ret;
2997
2998 WARN_ON(!con_secure(con));
2999
3000 ret = decrypt_control_remainder(con);
3001 if (ret) {
3002 if (ret == -EBADMSG)
3003 con->error_msg = "integrity error, bad control remainder auth tag";
3004 return ret;
3005 }
3006
3007 return __handle_control(con, p: con->v2.in_kvecs[0].iov_base -
3008 CEPH_PREAMBLE_INLINE_LEN);
3009}
3010
3011static int handle_epilogue(struct ceph_connection *con)
3012{
3013 u32 front_crc, middle_crc, data_crc;
3014 int ret;
3015
3016 if (con_secure(con)) {
3017 ret = decrypt_tail(con);
3018 if (ret) {
3019 if (ret == -EBADMSG)
3020 con->error_msg = "integrity error, bad epilogue auth tag";
3021 return ret;
3022 }
3023
3024 /* just late_status */
3025 ret = decode_epilogue(p: con->v2.in_buf, NULL, NULL, NULL);
3026 if (ret) {
3027 con->error_msg = "protocol error, bad epilogue";
3028 return ret;
3029 }
3030 } else {
3031 ret = decode_epilogue(p: con->v2.in_buf, front_crc: &front_crc,
3032 middle_crc: &middle_crc, data_crc: &data_crc);
3033 if (ret) {
3034 con->error_msg = "protocol error, bad epilogue";
3035 return ret;
3036 }
3037
3038 ret = verify_epilogue_crcs(con, front_crc, middle_crc,
3039 data_crc);
3040 if (ret) {
3041 con->error_msg = "integrity error, bad crc";
3042 return ret;
3043 }
3044 }
3045
3046 return process_message(con);
3047}
3048
3049static void finish_skip(struct ceph_connection *con)
3050{
3051 dout("%s con %p\n", __func__, con);
3052
3053 if (con_secure(con))
3054 gcm_inc_nonce(nonce: &con->v2.in_gcm_nonce);
3055
3056 __finish_skip(con);
3057}
3058
3059static int populate_in_iter(struct ceph_connection *con)
3060{
3061 int ret;
3062
3063 dout("%s con %p state %d in_state %d\n", __func__, con, con->state,
3064 con->v2.in_state);
3065 WARN_ON(iov_iter_count(&con->v2.in_iter));
3066
3067 if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) {
3068 ret = process_banner_prefix(con);
3069 } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) {
3070 ret = process_banner_payload(con);
3071 } else if ((con->state >= CEPH_CON_S_V2_HELLO &&
3072 con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) ||
3073 con->state == CEPH_CON_S_OPEN) {
3074 switch (con->v2.in_state) {
3075 case IN_S_HANDLE_PREAMBLE:
3076 ret = handle_preamble(con);
3077 break;
3078 case IN_S_HANDLE_CONTROL:
3079 ret = handle_control(con);
3080 break;
3081 case IN_S_HANDLE_CONTROL_REMAINDER:
3082 ret = handle_control_remainder(con);
3083 break;
3084 case IN_S_PREPARE_READ_DATA:
3085 ret = prepare_read_data(con);
3086 break;
3087 case IN_S_PREPARE_READ_DATA_CONT:
3088 prepare_read_data_cont(con);
3089 ret = 0;
3090 break;
3091 case IN_S_PREPARE_READ_ENC_PAGE:
3092 prepare_read_enc_page(con);
3093 ret = 0;
3094 break;
3095 case IN_S_PREPARE_SPARSE_DATA:
3096 ret = prepare_sparse_read_data(con);
3097 break;
3098 case IN_S_PREPARE_SPARSE_DATA_CONT:
3099 ret = prepare_sparse_read_cont(con);
3100 break;
3101 case IN_S_HANDLE_EPILOGUE:
3102 ret = handle_epilogue(con);
3103 break;
3104 case IN_S_FINISH_SKIP:
3105 finish_skip(con);
3106 ret = 0;
3107 break;
3108 default:
3109 WARN(1, "bad in_state %d", con->v2.in_state);
3110 return -EINVAL;
3111 }
3112 } else {
3113 WARN(1, "bad state %d", con->state);
3114 return -EINVAL;
3115 }
3116 if (ret) {
3117 dout("%s con %p error %d\n", __func__, con, ret);
3118 return ret;
3119 }
3120
3121 if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
3122 return -ENODATA;
3123 dout("%s con %p populated %zu\n", __func__, con,
3124 iov_iter_count(&con->v2.in_iter));
3125 return 1;
3126}
3127
3128int ceph_con_v2_try_read(struct ceph_connection *con)
3129{
3130 int ret;
3131
3132 dout("%s con %p state %d need %zu\n", __func__, con, con->state,
3133 iov_iter_count(&con->v2.in_iter));
3134
3135 if (con->state == CEPH_CON_S_PREOPEN)
3136 return 0;
3137
3138 /*
3139 * We should always have something pending here. If not,
3140 * avoid calling populate_in_iter() as if we read something
3141 * (ceph_tcp_recv() would immediately return 1).
3142 */
3143 if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
3144 return -ENODATA;
3145
3146 for (;;) {
3147 ret = ceph_tcp_recv(con);
3148 if (ret <= 0)
3149 return ret;
3150
3151 ret = populate_in_iter(con);
3152 if (ret <= 0) {
3153 if (ret && ret != -EAGAIN && !con->error_msg)
3154 con->error_msg = "read processing error";
3155 return ret;
3156 }
3157 }
3158}
3159
3160static void queue_data(struct ceph_connection *con, struct ceph_msg *msg)
3161{
3162 struct bio_vec bv;
3163
3164 con->v2.out_epil.data_crc = -1;
3165 ceph_msg_data_cursor_init(cursor: &con->v2.out_cursor, msg,
3166 length: data_len(msg));
3167
3168 get_bvec_at(cursor: &con->v2.out_cursor, bv: &bv);
3169 set_out_bvec(con, bv: &bv, zerocopy: true);
3170 con->v2.out_state = OUT_S_QUEUE_DATA_CONT;
3171}
3172
3173static void queue_data_cont(struct ceph_connection *con, struct ceph_msg *msg)
3174{
3175 struct bio_vec bv;
3176
3177 con->v2.out_epil.data_crc = ceph_crc32c_page(
3178 crc: con->v2.out_epil.data_crc, page: con->v2.out_bvec.bv_page,
3179 page_offset: con->v2.out_bvec.bv_offset, length: con->v2.out_bvec.bv_len);
3180
3181 ceph_msg_data_advance(cursor: &con->v2.out_cursor, bytes: con->v2.out_bvec.bv_len);
3182 if (con->v2.out_cursor.total_resid) {
3183 get_bvec_at(cursor: &con->v2.out_cursor, bv: &bv);
3184 set_out_bvec(con, bv: &bv, zerocopy: true);
3185 WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT);
3186 return;
3187 }
3188
3189 /*
3190 * We've written all data. Queue epilogue. Once it's written,
3191 * we are done.
3192 */
3193 reset_out_kvecs(con);
3194 prepare_epilogue_plain(con, msg, aborted: false);
3195 con->v2.out_state = OUT_S_FINISH_MESSAGE;
3196}
3197
3198static void queue_enc_page(struct ceph_connection *con)
3199{
3200 struct bio_vec bv;
3201
3202 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i,
3203 con->v2.out_enc_resid);
3204 WARN_ON(!con->v2.out_enc_resid);
3205
3206 bvec_set_page(bv: &bv, page: con->v2.out_enc_pages[con->v2.out_enc_i],
3207 min(con->v2.out_enc_resid, (int)PAGE_SIZE), offset: 0);
3208
3209 set_out_bvec(con, bv: &bv, zerocopy: false);
3210 con->v2.out_enc_i++;
3211 con->v2.out_enc_resid -= bv.bv_len;
3212
3213 if (con->v2.out_enc_resid) {
3214 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE);
3215 return;
3216 }
3217
3218 /*
3219 * We've queued the last piece of ciphertext (ending with
3220 * epilogue) + auth tag. Once it's written, we are done.
3221 */
3222 WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt);
3223 con->v2.out_state = OUT_S_FINISH_MESSAGE;
3224}
3225
3226static void queue_zeros(struct ceph_connection *con, struct ceph_msg *msg)
3227{
3228 dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero);
3229
3230 if (con->v2.out_zero) {
3231 set_out_bvec_zero(con);
3232 con->v2.out_zero -= con->v2.out_bvec.bv_len;
3233 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3234 return;
3235 }
3236
3237 /*
3238 * We've zero-filled everything up to epilogue. Queue epilogue
3239 * with late_status set to ABORTED and crcs adjusted for zeros.
3240 * Once it's written, we are done patching up for the revoke.
3241 */
3242 reset_out_kvecs(con);
3243 prepare_epilogue_plain(con, msg, aborted: true);
3244 con->v2.out_state = OUT_S_FINISH_MESSAGE;
3245}
3246
3247static void finish_message(struct ceph_connection *con)
3248{
3249 dout("%s con %p msg %p\n", __func__, con, con->out_msg);
3250
3251 /* we end up here both plain and secure modes */
3252 if (con->v2.out_enc_pages) {
3253 WARN_ON(!con->v2.out_enc_page_cnt);
3254 ceph_release_page_vector(pages: con->v2.out_enc_pages,
3255 num_pages: con->v2.out_enc_page_cnt);
3256 con->v2.out_enc_pages = NULL;
3257 con->v2.out_enc_page_cnt = 0;
3258 }
3259 /* message may have been revoked */
3260 if (con->out_msg) {
3261 ceph_msg_put(msg: con->out_msg);
3262 con->out_msg = NULL;
3263 }
3264
3265 con->v2.out_state = OUT_S_GET_NEXT;
3266}
3267
3268static int populate_out_iter(struct ceph_connection *con)
3269{
3270 struct ceph_msg *msg;
3271 int ret;
3272
3273 dout("%s con %p state %d out_state %d\n", __func__, con, con->state,
3274 con->v2.out_state);
3275 WARN_ON(iov_iter_count(&con->v2.out_iter));
3276
3277 if (con->state != CEPH_CON_S_OPEN) {
3278 WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX ||
3279 con->state > CEPH_CON_S_V2_SESSION_RECONNECT);
3280 goto nothing_pending;
3281 }
3282
3283 switch (con->v2.out_state) {
3284 case OUT_S_QUEUE_DATA:
3285 WARN_ON(!con->out_msg);
3286 queue_data(con, msg: con->out_msg);
3287 goto populated;
3288 case OUT_S_QUEUE_DATA_CONT:
3289 WARN_ON(!con->out_msg);
3290 queue_data_cont(con, msg: con->out_msg);
3291 goto populated;
3292 case OUT_S_QUEUE_ENC_PAGE:
3293 queue_enc_page(con);
3294 goto populated;
3295 case OUT_S_QUEUE_ZEROS:
3296 WARN_ON(con->out_msg); /* revoked */
3297 queue_zeros(con, msg: con->out_msg);
3298 goto populated;
3299 case OUT_S_FINISH_MESSAGE:
3300 finish_message(con);
3301 break;
3302 case OUT_S_GET_NEXT:
3303 break;
3304 default:
3305 WARN(1, "bad out_state %d", con->v2.out_state);
3306 return -EINVAL;
3307 }
3308
3309 WARN_ON(con->v2.out_state != OUT_S_GET_NEXT);
3310 if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) {
3311 ret = prepare_keepalive2(con);
3312 if (ret) {
3313 pr_err("prepare_keepalive2 failed: %d\n", ret);
3314 return ret;
3315 }
3316 } else if ((msg = ceph_con_get_out_msg(con)) != NULL) {
3317 ret = prepare_message(con, msg);
3318 if (ret) {
3319 pr_err("prepare_message failed: %d\n", ret);
3320 return ret;
3321 }
3322 } else if (con->in_seq > con->in_seq_acked) {
3323 ret = prepare_ack(con);
3324 if (ret) {
3325 pr_err("prepare_ack failed: %d\n", ret);
3326 return ret;
3327 }
3328 } else {
3329 goto nothing_pending;
3330 }
3331
3332populated:
3333 if (WARN_ON(!iov_iter_count(&con->v2.out_iter)))
3334 return -ENODATA;
3335 dout("%s con %p populated %zu\n", __func__, con,
3336 iov_iter_count(&con->v2.out_iter));
3337 return 1;
3338
3339nothing_pending:
3340 WARN_ON(iov_iter_count(&con->v2.out_iter));
3341 dout("%s con %p nothing pending\n", __func__, con);
3342 ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING);
3343 return 0;
3344}
3345
3346int ceph_con_v2_try_write(struct ceph_connection *con)
3347{
3348 int ret;
3349
3350 dout("%s con %p state %d have %zu\n", __func__, con, con->state,
3351 iov_iter_count(&con->v2.out_iter));
3352
3353 /* open the socket first? */
3354 if (con->state == CEPH_CON_S_PREOPEN) {
3355 WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2);
3356
3357 /*
3358 * Always bump global_seq. Bump connect_seq only if
3359 * there is a session (i.e. we are reconnecting and will
3360 * send session_reconnect instead of client_ident).
3361 */
3362 con->v2.global_seq = ceph_get_global_seq(msgr: con->msgr, gt: 0);
3363 if (con->v2.server_cookie)
3364 con->v2.connect_seq++;
3365
3366 ret = prepare_read_banner_prefix(con);
3367 if (ret) {
3368 pr_err("prepare_read_banner_prefix failed: %d\n", ret);
3369 con->error_msg = "connect error";
3370 return ret;
3371 }
3372
3373 reset_out_kvecs(con);
3374 ret = prepare_banner(con);
3375 if (ret) {
3376 pr_err("prepare_banner failed: %d\n", ret);
3377 con->error_msg = "connect error";
3378 return ret;
3379 }
3380
3381 ret = ceph_tcp_connect(con);
3382 if (ret) {
3383 pr_err("ceph_tcp_connect failed: %d\n", ret);
3384 con->error_msg = "connect error";
3385 return ret;
3386 }
3387 }
3388
3389 if (!iov_iter_count(i: &con->v2.out_iter)) {
3390 ret = populate_out_iter(con);
3391 if (ret <= 0) {
3392 if (ret && ret != -EAGAIN && !con->error_msg)
3393 con->error_msg = "write processing error";
3394 return ret;
3395 }
3396 }
3397
3398 tcp_sock_set_cork(sk: con->sock->sk, on: true);
3399 for (;;) {
3400 ret = ceph_tcp_send(con);
3401 if (ret <= 0)
3402 break;
3403
3404 ret = populate_out_iter(con);
3405 if (ret <= 0) {
3406 if (ret && ret != -EAGAIN && !con->error_msg)
3407 con->error_msg = "write processing error";
3408 break;
3409 }
3410 }
3411
3412 tcp_sock_set_cork(sk: con->sock->sk, on: false);
3413 return ret;
3414}
3415
3416static u32 crc32c_zeros(u32 crc, int zero_len)
3417{
3418 int len;
3419
3420 while (zero_len) {
3421 len = min(zero_len, (int)PAGE_SIZE);
3422 crc = crc32c(crc, page_address(ceph_zero_page), len);
3423 zero_len -= len;
3424 }
3425
3426 return crc;
3427}
3428
3429static void prepare_zero_front(struct ceph_connection *con,
3430 struct ceph_msg *msg, int resid)
3431{
3432 int sent;
3433
3434 WARN_ON(!resid || resid > front_len(msg));
3435 sent = front_len(msg) - resid;
3436 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3437
3438 if (sent) {
3439 con->v2.out_epil.front_crc =
3440 crc32c(crc: -1, p: msg->front.iov_base, len: sent);
3441 con->v2.out_epil.front_crc =
3442 crc32c_zeros(crc: con->v2.out_epil.front_crc, zero_len: resid);
3443 } else {
3444 con->v2.out_epil.front_crc = crc32c_zeros(crc: -1, zero_len: resid);
3445 }
3446
3447 con->v2.out_iter.count -= resid;
3448 out_zero_add(con, len: resid);
3449}
3450
3451static void prepare_zero_middle(struct ceph_connection *con,
3452 struct ceph_msg *msg, int resid)
3453{
3454 int sent;
3455
3456 WARN_ON(!resid || resid > middle_len(msg));
3457 sent = middle_len(msg) - resid;
3458 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3459
3460 if (sent) {
3461 con->v2.out_epil.middle_crc =
3462 crc32c(crc: -1, p: msg->middle->vec.iov_base, len: sent);
3463 con->v2.out_epil.middle_crc =
3464 crc32c_zeros(crc: con->v2.out_epil.middle_crc, zero_len: resid);
3465 } else {
3466 con->v2.out_epil.middle_crc = crc32c_zeros(crc: -1, zero_len: resid);
3467 }
3468
3469 con->v2.out_iter.count -= resid;
3470 out_zero_add(con, len: resid);
3471}
3472
3473static void prepare_zero_data(struct ceph_connection *con,
3474 struct ceph_msg *msg)
3475{
3476 dout("%s con %p\n", __func__, con);
3477 con->v2.out_epil.data_crc = crc32c_zeros(crc: -1, zero_len: data_len(msg));
3478 out_zero_add(con, len: data_len(msg));
3479}
3480
3481static void revoke_at_queue_data(struct ceph_connection *con,
3482 struct ceph_msg *msg)
3483{
3484 int boundary;
3485 int resid;
3486
3487 WARN_ON(!data_len(msg));
3488 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3489 resid = iov_iter_count(i: &con->v2.out_iter);
3490
3491 boundary = front_len(msg) + middle_len(msg);
3492 if (resid > boundary) {
3493 resid -= boundary;
3494 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3495 dout("%s con %p was sending head\n", __func__, con);
3496 if (front_len(msg))
3497 prepare_zero_front(con, msg, resid: front_len(msg));
3498 if (middle_len(msg))
3499 prepare_zero_middle(con, msg, resid: middle_len(msg));
3500 prepare_zero_data(con, msg);
3501 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3502 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3503 return;
3504 }
3505
3506 boundary = middle_len(msg);
3507 if (resid > boundary) {
3508 resid -= boundary;
3509 dout("%s con %p was sending front\n", __func__, con);
3510 prepare_zero_front(con, msg, resid);
3511 if (middle_len(msg))
3512 prepare_zero_middle(con, msg, resid: middle_len(msg));
3513 prepare_zero_data(con, msg);
3514 queue_zeros(con, msg);
3515 return;
3516 }
3517
3518 WARN_ON(!resid);
3519 dout("%s con %p was sending middle\n", __func__, con);
3520 prepare_zero_middle(con, msg, resid);
3521 prepare_zero_data(con, msg);
3522 queue_zeros(con, msg);
3523}
3524
3525static void revoke_at_queue_data_cont(struct ceph_connection *con,
3526 struct ceph_msg *msg)
3527{
3528 int sent, resid; /* current piece of data */
3529
3530 WARN_ON(!data_len(msg));
3531 WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter));
3532 resid = iov_iter_count(i: &con->v2.out_iter);
3533 WARN_ON(!resid || resid > con->v2.out_bvec.bv_len);
3534 sent = con->v2.out_bvec.bv_len - resid;
3535 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3536
3537 if (sent) {
3538 con->v2.out_epil.data_crc = ceph_crc32c_page(
3539 crc: con->v2.out_epil.data_crc, page: con->v2.out_bvec.bv_page,
3540 page_offset: con->v2.out_bvec.bv_offset, length: sent);
3541 ceph_msg_data_advance(cursor: &con->v2.out_cursor, bytes: sent);
3542 }
3543 WARN_ON(resid > con->v2.out_cursor.total_resid);
3544 con->v2.out_epil.data_crc = crc32c_zeros(crc: con->v2.out_epil.data_crc,
3545 zero_len: con->v2.out_cursor.total_resid);
3546
3547 con->v2.out_iter.count -= resid;
3548 out_zero_add(con, len: con->v2.out_cursor.total_resid);
3549 queue_zeros(con, msg);
3550}
3551
3552static void revoke_at_finish_message(struct ceph_connection *con,
3553 struct ceph_msg *msg)
3554{
3555 int boundary;
3556 int resid;
3557
3558 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3559 resid = iov_iter_count(i: &con->v2.out_iter);
3560
3561 if (!front_len(msg) && !middle_len(msg) &&
3562 !data_len(msg)) {
3563 WARN_ON(!resid || resid > MESSAGE_HEAD_PLAIN_LEN);
3564 dout("%s con %p was sending head (empty message) - noop\n",
3565 __func__, con);
3566 return;
3567 }
3568
3569 boundary = front_len(msg) + middle_len(msg) +
3570 CEPH_EPILOGUE_PLAIN_LEN;
3571 if (resid > boundary) {
3572 resid -= boundary;
3573 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3574 dout("%s con %p was sending head\n", __func__, con);
3575 if (front_len(msg))
3576 prepare_zero_front(con, msg, resid: front_len(msg));
3577 if (middle_len(msg))
3578 prepare_zero_middle(con, msg, resid: middle_len(msg));
3579 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3580 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3581 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3582 return;
3583 }
3584
3585 boundary = middle_len(msg) + CEPH_EPILOGUE_PLAIN_LEN;
3586 if (resid > boundary) {
3587 resid -= boundary;
3588 dout("%s con %p was sending front\n", __func__, con);
3589 prepare_zero_front(con, msg, resid);
3590 if (middle_len(msg))
3591 prepare_zero_middle(con, msg, resid: middle_len(msg));
3592 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3593 queue_zeros(con, msg);
3594 return;
3595 }
3596
3597 boundary = CEPH_EPILOGUE_PLAIN_LEN;
3598 if (resid > boundary) {
3599 resid -= boundary;
3600 dout("%s con %p was sending middle\n", __func__, con);
3601 prepare_zero_middle(con, msg, resid);
3602 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3603 queue_zeros(con, msg);
3604 return;
3605 }
3606
3607 WARN_ON(!resid);
3608 dout("%s con %p was sending epilogue - noop\n", __func__, con);
3609}
3610
3611void ceph_con_v2_revoke(struct ceph_connection *con, struct ceph_msg *msg)
3612{
3613 WARN_ON(con->v2.out_zero);
3614
3615 if (con_secure(con)) {
3616 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE &&
3617 con->v2.out_state != OUT_S_FINISH_MESSAGE);
3618 dout("%s con %p secure - noop\n", __func__, con);
3619 return;
3620 }
3621
3622 switch (con->v2.out_state) {
3623 case OUT_S_QUEUE_DATA:
3624 revoke_at_queue_data(con, msg);
3625 break;
3626 case OUT_S_QUEUE_DATA_CONT:
3627 revoke_at_queue_data_cont(con, msg);
3628 break;
3629 case OUT_S_FINISH_MESSAGE:
3630 revoke_at_finish_message(con, msg);
3631 break;
3632 default:
3633 WARN(1, "bad out_state %d", con->v2.out_state);
3634 break;
3635 }
3636}
3637
3638static void revoke_at_prepare_read_data(struct ceph_connection *con)
3639{
3640 int remaining;
3641 int resid;
3642
3643 WARN_ON(con_secure(con));
3644 WARN_ON(!data_len(con->in_msg));
3645 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
3646 resid = iov_iter_count(i: &con->v2.in_iter);
3647 WARN_ON(!resid);
3648
3649 remaining = data_len(msg: con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3650 dout("%s con %p resid %d remaining %d\n", __func__, con, resid,
3651 remaining);
3652 con->v2.in_iter.count -= resid;
3653 set_in_skip(con, len: resid + remaining);
3654 con->v2.in_state = IN_S_FINISH_SKIP;
3655}
3656
3657static void revoke_at_prepare_read_data_cont(struct ceph_connection *con)
3658{
3659 int recved, resid; /* current piece of data */
3660 int remaining;
3661
3662 WARN_ON(con_secure(con));
3663 WARN_ON(!data_len(con->in_msg));
3664 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3665 resid = iov_iter_count(i: &con->v2.in_iter);
3666 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3667 recved = con->v2.in_bvec.bv_len - resid;
3668 dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid);
3669
3670 if (recved)
3671 ceph_msg_data_advance(cursor: &con->v2.in_cursor, bytes: recved);
3672 WARN_ON(resid > con->v2.in_cursor.total_resid);
3673
3674 remaining = CEPH_EPILOGUE_PLAIN_LEN;
3675 dout("%s con %p total_resid %zu remaining %d\n", __func__, con,
3676 con->v2.in_cursor.total_resid, remaining);
3677 con->v2.in_iter.count -= resid;
3678 set_in_skip(con, len: con->v2.in_cursor.total_resid + remaining);
3679 con->v2.in_state = IN_S_FINISH_SKIP;
3680}
3681
3682static void revoke_at_prepare_read_enc_page(struct ceph_connection *con)
3683{
3684 int resid; /* current enc page (not necessarily data) */
3685
3686 WARN_ON(!con_secure(con));
3687 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3688 resid = iov_iter_count(i: &con->v2.in_iter);
3689 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3690
3691 dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid,
3692 con->v2.in_enc_resid);
3693 con->v2.in_iter.count -= resid;
3694 set_in_skip(con, len: resid + con->v2.in_enc_resid);
3695 con->v2.in_state = IN_S_FINISH_SKIP;
3696}
3697
3698static void revoke_at_prepare_sparse_data(struct ceph_connection *con)
3699{
3700 int resid; /* current piece of data */
3701 int remaining;
3702
3703 WARN_ON(con_secure(con));
3704 WARN_ON(!data_len(con->in_msg));
3705 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3706 resid = iov_iter_count(i: &con->v2.in_iter);
3707 dout("%s con %p resid %d\n", __func__, con, resid);
3708
3709 remaining = CEPH_EPILOGUE_PLAIN_LEN + con->v2.data_len_remain;
3710 con->v2.in_iter.count -= resid;
3711 set_in_skip(con, len: resid + remaining);
3712 con->v2.in_state = IN_S_FINISH_SKIP;
3713}
3714
3715static void revoke_at_handle_epilogue(struct ceph_connection *con)
3716{
3717 int resid;
3718
3719 resid = iov_iter_count(i: &con->v2.in_iter);
3720 WARN_ON(!resid);
3721
3722 dout("%s con %p resid %d\n", __func__, con, resid);
3723 con->v2.in_iter.count -= resid;
3724 set_in_skip(con, len: resid);
3725 con->v2.in_state = IN_S_FINISH_SKIP;
3726}
3727
3728void ceph_con_v2_revoke_incoming(struct ceph_connection *con)
3729{
3730 switch (con->v2.in_state) {
3731 case IN_S_PREPARE_SPARSE_DATA:
3732 case IN_S_PREPARE_READ_DATA:
3733 revoke_at_prepare_read_data(con);
3734 break;
3735 case IN_S_PREPARE_READ_DATA_CONT:
3736 revoke_at_prepare_read_data_cont(con);
3737 break;
3738 case IN_S_PREPARE_READ_ENC_PAGE:
3739 revoke_at_prepare_read_enc_page(con);
3740 break;
3741 case IN_S_PREPARE_SPARSE_DATA_CONT:
3742 revoke_at_prepare_sparse_data(con);
3743 break;
3744 case IN_S_HANDLE_EPILOGUE:
3745 revoke_at_handle_epilogue(con);
3746 break;
3747 default:
3748 WARN(1, "bad in_state %d", con->v2.in_state);
3749 break;
3750 }
3751}
3752
3753bool ceph_con_v2_opened(struct ceph_connection *con)
3754{
3755 return con->v2.peer_global_seq;
3756}
3757
3758void ceph_con_v2_reset_session(struct ceph_connection *con)
3759{
3760 con->v2.client_cookie = 0;
3761 con->v2.server_cookie = 0;
3762 con->v2.global_seq = 0;
3763 con->v2.connect_seq = 0;
3764 con->v2.peer_global_seq = 0;
3765}
3766
3767void ceph_con_v2_reset_protocol(struct ceph_connection *con)
3768{
3769 iov_iter_truncate(i: &con->v2.in_iter, count: 0);
3770 iov_iter_truncate(i: &con->v2.out_iter, count: 0);
3771 con->v2.out_zero = 0;
3772
3773 clear_in_sign_kvecs(con);
3774 clear_out_sign_kvecs(con);
3775 free_conn_bufs(con);
3776
3777 if (con->v2.in_enc_pages) {
3778 WARN_ON(!con->v2.in_enc_page_cnt);
3779 ceph_release_page_vector(pages: con->v2.in_enc_pages,
3780 num_pages: con->v2.in_enc_page_cnt);
3781 con->v2.in_enc_pages = NULL;
3782 con->v2.in_enc_page_cnt = 0;
3783 }
3784 if (con->v2.out_enc_pages) {
3785 WARN_ON(!con->v2.out_enc_page_cnt);
3786 ceph_release_page_vector(pages: con->v2.out_enc_pages,
3787 num_pages: con->v2.out_enc_page_cnt);
3788 con->v2.out_enc_pages = NULL;
3789 con->v2.out_enc_page_cnt = 0;
3790 }
3791
3792 con->v2.con_mode = CEPH_CON_MODE_UNKNOWN;
3793 memzero_explicit(s: &con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN);
3794 memzero_explicit(s: &con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN);
3795
3796 memzero_explicit(s: &con->v2.hmac_key, count: sizeof(con->v2.hmac_key));
3797 con->v2.hmac_key_set = false;
3798 if (con->v2.gcm_req) {
3799 aead_request_free(req: con->v2.gcm_req);
3800 con->v2.gcm_req = NULL;
3801 }
3802 if (con->v2.gcm_tfm) {
3803 crypto_free_aead(tfm: con->v2.gcm_tfm);
3804 con->v2.gcm_tfm = NULL;
3805 }
3806}
3807

source code of linux/net/ceph/messenger_v2.c