1// SPDX-License-Identifier: GPL-2.0
2/*
3 * t10_pi.c - Functions for generating and verifying T10 Protection
4 * Information.
5 */
6
7#include <linux/t10-pi.h>
8#include <linux/blk-integrity.h>
9#include <linux/crc-t10dif.h>
10#include <linux/crc64.h>
11#include <net/checksum.h>
12#include <linux/unaligned.h>
13#include "blk.h"
14
15struct blk_integrity_iter {
16 void *prot_buf;
17 void *data_buf;
18 sector_t seed;
19 unsigned int data_size;
20 unsigned short interval;
21 const char *disk_name;
22};
23
24static __be16 t10_pi_csum(__be16 csum, void *data, unsigned int len,
25 unsigned char csum_type)
26{
27 if (csum_type == BLK_INTEGRITY_CSUM_IP)
28 return (__force __be16)ip_compute_csum(buff: data, len);
29 return cpu_to_be16(crc_t10dif_update(be16_to_cpu(csum), data, len));
30}
31
32/*
33 * Type 1 and Type 2 protection use the same format: 16 bit guard tag,
34 * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref
35 * tag.
36 */
37static void t10_pi_generate(struct blk_integrity_iter *iter,
38 struct blk_integrity *bi)
39{
40 u8 offset = bi->pi_offset;
41 unsigned int i;
42
43 for (i = 0 ; i < iter->data_size ; i += iter->interval) {
44 struct t10_pi_tuple *pi = iter->prot_buf + offset;
45
46 pi->guard_tag = t10_pi_csum(csum: 0, data: iter->data_buf, len: iter->interval,
47 csum_type: bi->csum_type);
48 if (offset)
49 pi->guard_tag = t10_pi_csum(csum: pi->guard_tag,
50 data: iter->prot_buf, len: offset, csum_type: bi->csum_type);
51 pi->app_tag = 0;
52
53 if (bi->flags & BLK_INTEGRITY_REF_TAG)
54 pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed));
55 else
56 pi->ref_tag = 0;
57
58 iter->data_buf += iter->interval;
59 iter->prot_buf += bi->tuple_size;
60 iter->seed++;
61 }
62}
63
64static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter,
65 struct blk_integrity *bi)
66{
67 u8 offset = bi->pi_offset;
68 unsigned int i;
69
70 for (i = 0 ; i < iter->data_size ; i += iter->interval) {
71 struct t10_pi_tuple *pi = iter->prot_buf + offset;
72 __be16 csum;
73
74 if (bi->flags & BLK_INTEGRITY_REF_TAG) {
75 if (pi->app_tag == T10_PI_APP_ESCAPE)
76 goto next;
77
78 if (be32_to_cpu(pi->ref_tag) !=
79 lower_32_bits(iter->seed)) {
80 pr_err("%s: ref tag error at location %llu " \
81 "(rcvd %u)\n", iter->disk_name,
82 (unsigned long long)
83 iter->seed, be32_to_cpu(pi->ref_tag));
84 return BLK_STS_PROTECTION;
85 }
86 } else {
87 if (pi->app_tag == T10_PI_APP_ESCAPE &&
88 pi->ref_tag == T10_PI_REF_ESCAPE)
89 goto next;
90 }
91
92 csum = t10_pi_csum(csum: 0, data: iter->data_buf, len: iter->interval,
93 csum_type: bi->csum_type);
94 if (offset)
95 csum = t10_pi_csum(csum, data: iter->prot_buf, len: offset,
96 csum_type: bi->csum_type);
97
98 if (pi->guard_tag != csum) {
99 pr_err("%s: guard tag error at sector %llu " \
100 "(rcvd %04x, want %04x)\n", iter->disk_name,
101 (unsigned long long)iter->seed,
102 be16_to_cpu(pi->guard_tag), be16_to_cpu(csum));
103 return BLK_STS_PROTECTION;
104 }
105
106next:
107 iter->data_buf += iter->interval;
108 iter->prot_buf += bi->tuple_size;
109 iter->seed++;
110 }
111
112 return BLK_STS_OK;
113}
114
115/**
116 * t10_pi_type1_prepare - prepare PI prior submitting request to device
117 * @rq: request with PI that should be prepared
118 *
119 * For Type 1/Type 2, the virtual start sector is the one that was
120 * originally submitted by the block layer for the ref_tag usage. Due to
121 * partitioning, MD/DM cloning, etc. the actual physical start sector is
122 * likely to be different. Remap protection information to match the
123 * physical LBA.
124 */
125static void t10_pi_type1_prepare(struct request *rq)
126{
127 struct blk_integrity *bi = &rq->q->limits.integrity;
128 const int tuple_sz = bi->tuple_size;
129 u32 ref_tag = t10_pi_ref_tag(rq);
130 u8 offset = bi->pi_offset;
131 struct bio *bio;
132
133 __rq_for_each_bio(bio, rq) {
134 struct bio_integrity_payload *bip = bio_integrity(bio);
135 u32 virt = bip_get_seed(bip) & 0xffffffff;
136 struct bio_vec iv;
137 struct bvec_iter iter;
138
139 /* Already remapped? */
140 if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
141 break;
142
143 bip_for_each_vec(iv, bip, iter) {
144 unsigned int j;
145 void *p;
146
147 p = bvec_kmap_local(bvec: &iv);
148 for (j = 0; j < iv.bv_len; j += tuple_sz) {
149 struct t10_pi_tuple *pi = p + offset;
150
151 if (be32_to_cpu(pi->ref_tag) == virt)
152 pi->ref_tag = cpu_to_be32(ref_tag);
153 virt++;
154 ref_tag++;
155 p += tuple_sz;
156 }
157 kunmap_local(p);
158 }
159
160 bip->bip_flags |= BIP_MAPPED_INTEGRITY;
161 }
162}
163
164/**
165 * t10_pi_type1_complete - prepare PI prior returning request to the blk layer
166 * @rq: request with PI that should be prepared
167 * @nr_bytes: total bytes to prepare
168 *
169 * For Type 1/Type 2, the virtual start sector is the one that was
170 * originally submitted by the block layer for the ref_tag usage. Due to
171 * partitioning, MD/DM cloning, etc. the actual physical start sector is
172 * likely to be different. Since the physical start sector was submitted
173 * to the device, we should remap it back to virtual values expected by the
174 * block layer.
175 */
176static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
177{
178 struct blk_integrity *bi = &rq->q->limits.integrity;
179 unsigned intervals = nr_bytes >> bi->interval_exp;
180 const int tuple_sz = bi->tuple_size;
181 u32 ref_tag = t10_pi_ref_tag(rq);
182 u8 offset = bi->pi_offset;
183 struct bio *bio;
184
185 __rq_for_each_bio(bio, rq) {
186 struct bio_integrity_payload *bip = bio_integrity(bio);
187 u32 virt = bip_get_seed(bip) & 0xffffffff;
188 struct bio_vec iv;
189 struct bvec_iter iter;
190
191 bip_for_each_vec(iv, bip, iter) {
192 unsigned int j;
193 void *p;
194
195 p = bvec_kmap_local(bvec: &iv);
196 for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
197 struct t10_pi_tuple *pi = p + offset;
198
199 if (be32_to_cpu(pi->ref_tag) == ref_tag)
200 pi->ref_tag = cpu_to_be32(virt);
201 virt++;
202 ref_tag++;
203 intervals--;
204 p += tuple_sz;
205 }
206 kunmap_local(p);
207 }
208 }
209}
210
211static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len)
212{
213 return cpu_to_be64(crc64_nvme(crc, data, len));
214}
215
216static void ext_pi_crc64_generate(struct blk_integrity_iter *iter,
217 struct blk_integrity *bi)
218{
219 u8 offset = bi->pi_offset;
220 unsigned int i;
221
222 for (i = 0 ; i < iter->data_size ; i += iter->interval) {
223 struct crc64_pi_tuple *pi = iter->prot_buf + offset;
224
225 pi->guard_tag = ext_pi_crc64(crc: 0, data: iter->data_buf, len: iter->interval);
226 if (offset)
227 pi->guard_tag = ext_pi_crc64(be64_to_cpu(pi->guard_tag),
228 data: iter->prot_buf, len: offset);
229 pi->app_tag = 0;
230
231 if (bi->flags & BLK_INTEGRITY_REF_TAG)
232 put_unaligned_be48(val: iter->seed, p: pi->ref_tag);
233 else
234 put_unaligned_be48(val: 0ULL, p: pi->ref_tag);
235
236 iter->data_buf += iter->interval;
237 iter->prot_buf += bi->tuple_size;
238 iter->seed++;
239 }
240}
241
242static bool ext_pi_ref_escape(const u8 ref_tag[6])
243{
244 static const u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
245
246 return memcmp(p: ref_tag, q: ref_escape, size: sizeof(ref_escape)) == 0;
247}
248
249static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter,
250 struct blk_integrity *bi)
251{
252 u8 offset = bi->pi_offset;
253 unsigned int i;
254
255 for (i = 0; i < iter->data_size; i += iter->interval) {
256 struct crc64_pi_tuple *pi = iter->prot_buf + offset;
257 u64 ref, seed;
258 __be64 csum;
259
260 if (bi->flags & BLK_INTEGRITY_REF_TAG) {
261 if (pi->app_tag == T10_PI_APP_ESCAPE)
262 goto next;
263
264 ref = get_unaligned_be48(p: pi->ref_tag);
265 seed = lower_48_bits(n: iter->seed);
266 if (ref != seed) {
267 pr_err("%s: ref tag error at location %llu (rcvd %llu)\n",
268 iter->disk_name, seed, ref);
269 return BLK_STS_PROTECTION;
270 }
271 } else {
272 if (pi->app_tag == T10_PI_APP_ESCAPE &&
273 ext_pi_ref_escape(ref_tag: pi->ref_tag))
274 goto next;
275 }
276
277 csum = ext_pi_crc64(crc: 0, data: iter->data_buf, len: iter->interval);
278 if (offset)
279 csum = ext_pi_crc64(be64_to_cpu(csum), data: iter->prot_buf,
280 len: offset);
281
282 if (pi->guard_tag != csum) {
283 pr_err("%s: guard tag error at sector %llu " \
284 "(rcvd %016llx, want %016llx)\n",
285 iter->disk_name, (unsigned long long)iter->seed,
286 be64_to_cpu(pi->guard_tag), be64_to_cpu(csum));
287 return BLK_STS_PROTECTION;
288 }
289
290next:
291 iter->data_buf += iter->interval;
292 iter->prot_buf += bi->tuple_size;
293 iter->seed++;
294 }
295
296 return BLK_STS_OK;
297}
298
299static void ext_pi_type1_prepare(struct request *rq)
300{
301 struct blk_integrity *bi = &rq->q->limits.integrity;
302 const int tuple_sz = bi->tuple_size;
303 u64 ref_tag = ext_pi_ref_tag(rq);
304 u8 offset = bi->pi_offset;
305 struct bio *bio;
306
307 __rq_for_each_bio(bio, rq) {
308 struct bio_integrity_payload *bip = bio_integrity(bio);
309 u64 virt = lower_48_bits(n: bip_get_seed(bip));
310 struct bio_vec iv;
311 struct bvec_iter iter;
312
313 /* Already remapped? */
314 if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
315 break;
316
317 bip_for_each_vec(iv, bip, iter) {
318 unsigned int j;
319 void *p;
320
321 p = bvec_kmap_local(bvec: &iv);
322 for (j = 0; j < iv.bv_len; j += tuple_sz) {
323 struct crc64_pi_tuple *pi = p + offset;
324 u64 ref = get_unaligned_be48(p: pi->ref_tag);
325
326 if (ref == virt)
327 put_unaligned_be48(val: ref_tag, p: pi->ref_tag);
328 virt++;
329 ref_tag++;
330 p += tuple_sz;
331 }
332 kunmap_local(p);
333 }
334
335 bip->bip_flags |= BIP_MAPPED_INTEGRITY;
336 }
337}
338
339static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
340{
341 struct blk_integrity *bi = &rq->q->limits.integrity;
342 unsigned intervals = nr_bytes >> bi->interval_exp;
343 const int tuple_sz = bi->tuple_size;
344 u64 ref_tag = ext_pi_ref_tag(rq);
345 u8 offset = bi->pi_offset;
346 struct bio *bio;
347
348 __rq_for_each_bio(bio, rq) {
349 struct bio_integrity_payload *bip = bio_integrity(bio);
350 u64 virt = lower_48_bits(n: bip_get_seed(bip));
351 struct bio_vec iv;
352 struct bvec_iter iter;
353
354 bip_for_each_vec(iv, bip, iter) {
355 unsigned int j;
356 void *p;
357
358 p = bvec_kmap_local(bvec: &iv);
359 for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
360 struct crc64_pi_tuple *pi = p + offset;
361 u64 ref = get_unaligned_be48(p: pi->ref_tag);
362
363 if (ref == ref_tag)
364 put_unaligned_be48(val: virt, p: pi->ref_tag);
365 virt++;
366 ref_tag++;
367 intervals--;
368 p += tuple_sz;
369 }
370 kunmap_local(p);
371 }
372 }
373}
374
375void blk_integrity_generate(struct bio *bio)
376{
377 struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk);
378 struct bio_integrity_payload *bip = bio_integrity(bio);
379 struct blk_integrity_iter iter;
380 struct bvec_iter bviter;
381 struct bio_vec bv;
382
383 iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
384 iter.interval = 1 << bi->interval_exp;
385 iter.seed = bio->bi_iter.bi_sector;
386 iter.prot_buf = bvec_virt(bvec: bip->bip_vec);
387 bio_for_each_segment(bv, bio, bviter) {
388 void *kaddr = bvec_kmap_local(bvec: &bv);
389
390 iter.data_buf = kaddr;
391 iter.data_size = bv.bv_len;
392 switch (bi->csum_type) {
393 case BLK_INTEGRITY_CSUM_CRC64:
394 ext_pi_crc64_generate(iter: &iter, bi);
395 break;
396 case BLK_INTEGRITY_CSUM_CRC:
397 case BLK_INTEGRITY_CSUM_IP:
398 t10_pi_generate(iter: &iter, bi);
399 break;
400 default:
401 break;
402 }
403 kunmap_local(kaddr);
404 }
405}
406
407void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter)
408{
409 struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk);
410 struct bio_integrity_payload *bip = bio_integrity(bio);
411 struct blk_integrity_iter iter;
412 struct bvec_iter bviter;
413 struct bio_vec bv;
414
415 /*
416 * At the moment verify is called bi_iter has been advanced during split
417 * and completion, so use the copy created during submission here.
418 */
419 iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
420 iter.interval = 1 << bi->interval_exp;
421 iter.seed = saved_iter->bi_sector;
422 iter.prot_buf = bvec_virt(bvec: bip->bip_vec);
423 __bio_for_each_segment(bv, bio, bviter, *saved_iter) {
424 void *kaddr = bvec_kmap_local(bvec: &bv);
425 blk_status_t ret = BLK_STS_OK;
426
427 iter.data_buf = kaddr;
428 iter.data_size = bv.bv_len;
429 switch (bi->csum_type) {
430 case BLK_INTEGRITY_CSUM_CRC64:
431 ret = ext_pi_crc64_verify(iter: &iter, bi);
432 break;
433 case BLK_INTEGRITY_CSUM_CRC:
434 case BLK_INTEGRITY_CSUM_IP:
435 ret = t10_pi_verify(iter: &iter, bi);
436 break;
437 default:
438 break;
439 }
440 kunmap_local(kaddr);
441
442 if (ret) {
443 bio->bi_status = ret;
444 return;
445 }
446 }
447}
448
449void blk_integrity_prepare(struct request *rq)
450{
451 struct blk_integrity *bi = &rq->q->limits.integrity;
452
453 if (!(bi->flags & BLK_INTEGRITY_REF_TAG))
454 return;
455
456 if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)
457 ext_pi_type1_prepare(rq);
458 else
459 t10_pi_type1_prepare(rq);
460}
461
462void blk_integrity_complete(struct request *rq, unsigned int nr_bytes)
463{
464 struct blk_integrity *bi = &rq->q->limits.integrity;
465
466 if (!(bi->flags & BLK_INTEGRITY_REF_TAG))
467 return;
468
469 if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)
470 ext_pi_type1_complete(rq, nr_bytes);
471 else
472 t10_pi_type1_complete(rq, nr_bytes);
473}
474

source code of linux/block/t10-pi.c