1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2007 Oracle. All rights reserved.
4 */
5
6#include <linux/bio.h>
7#include <linux/slab.h>
8#include <linux/pagemap.h>
9#include <linux/highmem.h>
10#include <linux/sched/mm.h>
11#include <crypto/hash.h>
12#include "messages.h"
13#include "ctree.h"
14#include "disk-io.h"
15#include "transaction.h"
16#include "bio.h"
17#include "compression.h"
18#include "fs.h"
19#include "accessors.h"
20#include "file-item.h"
21#include "volumes.h"
22
23#define __MAX_CSUM_ITEMS(r, size) ((unsigned long)(((BTRFS_LEAF_DATA_SIZE(r) - \
24 sizeof(struct btrfs_item) * 2) / \
25 size) - 1))
26
27#define MAX_CSUM_ITEMS(r, size) (min_t(u32, __MAX_CSUM_ITEMS(r, size), \
28 PAGE_SIZE))
29
30/*
31 * Set inode's size according to filesystem options.
32 *
33 * @inode: inode we want to update the disk_i_size for
34 * @new_i_size: i_size we want to set to, 0 if we use i_size
35 *
36 * With NO_HOLES set this simply sets the disk_is_size to whatever i_size_read()
37 * returns as it is perfectly fine with a file that has holes without hole file
38 * extent items.
39 *
40 * However without NO_HOLES we need to only return the area that is contiguous
41 * from the 0 offset of the file. Otherwise we could end up adjust i_size up
42 * to an extent that has a gap in between.
43 *
44 * Finally new_i_size should only be set in the case of truncate where we're not
45 * ready to use i_size_read() as the limiter yet.
46 */
47void btrfs_inode_safe_disk_i_size_write(struct btrfs_inode *inode, u64 new_i_size)
48{
49 u64 start, end, i_size;
50 bool found;
51
52 spin_lock(lock: &inode->lock);
53 i_size = new_i_size ?: i_size_read(inode: &inode->vfs_inode);
54 if (!inode->file_extent_tree) {
55 inode->disk_i_size = i_size;
56 goto out_unlock;
57 }
58
59 found = btrfs_find_contiguous_extent_bit(tree: inode->file_extent_tree, start: 0, start_ret: &start,
60 end_ret: &end, bits: EXTENT_DIRTY);
61 if (found && start == 0)
62 i_size = min(i_size, end + 1);
63 else
64 i_size = 0;
65 inode->disk_i_size = i_size;
66out_unlock:
67 spin_unlock(lock: &inode->lock);
68}
69
70/*
71 * Mark range within a file as having a new extent inserted.
72 *
73 * @inode: inode being modified
74 * @start: start file offset of the file extent we've inserted
75 * @len: logical length of the file extent item
76 *
77 * Call when we are inserting a new file extent where there was none before.
78 * Does not need to call this in the case where we're replacing an existing file
79 * extent, however if not sure it's fine to call this multiple times.
80 *
81 * The start and len must match the file extent item, so thus must be sectorsize
82 * aligned.
83 */
84int btrfs_inode_set_file_extent_range(struct btrfs_inode *inode, u64 start,
85 u64 len)
86{
87 if (!inode->file_extent_tree)
88 return 0;
89
90 if (len == 0)
91 return 0;
92
93 ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize));
94
95 return btrfs_set_extent_bit(tree: inode->file_extent_tree, start, end: start + len - 1,
96 bits: EXTENT_DIRTY, NULL);
97}
98
99/*
100 * Mark an inode range as not having a backing extent.
101 *
102 * @inode: inode being modified
103 * @start: start file offset of the file extent we've inserted
104 * @len: logical length of the file extent item
105 *
106 * Called when we drop a file extent, for example when we truncate. Doesn't
107 * need to be called for cases where we're replacing a file extent, like when
108 * we've COWed a file extent.
109 *
110 * The start and len must match the file extent item, so thus must be sectorsize
111 * aligned.
112 */
113int btrfs_inode_clear_file_extent_range(struct btrfs_inode *inode, u64 start,
114 u64 len)
115{
116 if (!inode->file_extent_tree)
117 return 0;
118
119 if (len == 0)
120 return 0;
121
122 ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize) ||
123 len == (u64)-1);
124
125 return btrfs_clear_extent_bit(tree: inode->file_extent_tree, start,
126 end: start + len - 1, bits: EXTENT_DIRTY, NULL);
127}
128
129static size_t bytes_to_csum_size(const struct btrfs_fs_info *fs_info, u32 bytes)
130{
131 ASSERT(IS_ALIGNED(bytes, fs_info->sectorsize));
132
133 return (bytes >> fs_info->sectorsize_bits) * fs_info->csum_size;
134}
135
136static size_t csum_size_to_bytes(const struct btrfs_fs_info *fs_info, u32 csum_size)
137{
138 ASSERT(IS_ALIGNED(csum_size, fs_info->csum_size));
139
140 return (csum_size / fs_info->csum_size) << fs_info->sectorsize_bits;
141}
142
143static inline u32 max_ordered_sum_bytes(const struct btrfs_fs_info *fs_info)
144{
145 u32 max_csum_size = round_down(PAGE_SIZE - sizeof(struct btrfs_ordered_sum),
146 fs_info->csum_size);
147
148 return csum_size_to_bytes(fs_info, csum_size: max_csum_size);
149}
150
151/*
152 * Calculate the total size needed to allocate for an ordered sum structure
153 * spanning @bytes in the file.
154 */
155static int btrfs_ordered_sum_size(const struct btrfs_fs_info *fs_info, unsigned long bytes)
156{
157 return sizeof(struct btrfs_ordered_sum) + bytes_to_csum_size(fs_info, bytes);
158}
159
160int btrfs_insert_hole_extent(struct btrfs_trans_handle *trans,
161 struct btrfs_root *root,
162 u64 objectid, u64 pos, u64 num_bytes)
163{
164 int ret = 0;
165 struct btrfs_file_extent_item *item;
166 struct btrfs_key file_key;
167 BTRFS_PATH_AUTO_FREE(path);
168 struct extent_buffer *leaf;
169
170 path = btrfs_alloc_path();
171 if (!path)
172 return -ENOMEM;
173
174 file_key.objectid = objectid;
175 file_key.type = BTRFS_EXTENT_DATA_KEY;
176 file_key.offset = pos;
177
178 ret = btrfs_insert_empty_item(trans, root, path, key: &file_key,
179 data_size: sizeof(*item));
180 if (ret < 0)
181 return ret;
182 leaf = path->nodes[0];
183 item = btrfs_item_ptr(leaf, path->slots[0],
184 struct btrfs_file_extent_item);
185 btrfs_set_file_extent_disk_bytenr(eb: leaf, s: item, val: 0);
186 btrfs_set_file_extent_disk_num_bytes(eb: leaf, s: item, val: 0);
187 btrfs_set_file_extent_offset(eb: leaf, s: item, val: 0);
188 btrfs_set_file_extent_num_bytes(eb: leaf, s: item, val: num_bytes);
189 btrfs_set_file_extent_ram_bytes(eb: leaf, s: item, val: num_bytes);
190 btrfs_set_file_extent_generation(eb: leaf, s: item, val: trans->transid);
191 btrfs_set_file_extent_type(eb: leaf, s: item, val: BTRFS_FILE_EXTENT_REG);
192 btrfs_set_file_extent_compression(eb: leaf, s: item, val: 0);
193 btrfs_set_file_extent_encryption(eb: leaf, s: item, val: 0);
194 btrfs_set_file_extent_other_encoding(eb: leaf, s: item, val: 0);
195
196 return ret;
197}
198
199static struct btrfs_csum_item *
200btrfs_lookup_csum(struct btrfs_trans_handle *trans,
201 struct btrfs_root *root,
202 struct btrfs_path *path,
203 u64 bytenr, int cow)
204{
205 struct btrfs_fs_info *fs_info = root->fs_info;
206 int ret;
207 struct btrfs_key file_key;
208 struct btrfs_key found_key;
209 struct btrfs_csum_item *item;
210 struct extent_buffer *leaf;
211 u64 csum_offset = 0;
212 const u32 csum_size = fs_info->csum_size;
213 int csums_in_item;
214
215 file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
216 file_key.type = BTRFS_EXTENT_CSUM_KEY;
217 file_key.offset = bytenr;
218 ret = btrfs_search_slot(trans, root, key: &file_key, p: path, ins_len: 0, cow);
219 if (ret < 0)
220 goto fail;
221 leaf = path->nodes[0];
222 if (ret > 0) {
223 ret = 1;
224 if (path->slots[0] == 0)
225 goto fail;
226 path->slots[0]--;
227 btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
228 if (found_key.type != BTRFS_EXTENT_CSUM_KEY)
229 goto fail;
230
231 csum_offset = (bytenr - found_key.offset) >>
232 fs_info->sectorsize_bits;
233 csums_in_item = btrfs_item_size(eb: leaf, slot: path->slots[0]);
234 csums_in_item /= csum_size;
235
236 if (csum_offset == csums_in_item) {
237 ret = -EFBIG;
238 goto fail;
239 } else if (csum_offset > csums_in_item) {
240 goto fail;
241 }
242 }
243 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);
244 item = (struct btrfs_csum_item *)((unsigned char *)item +
245 csum_offset * csum_size);
246 return item;
247fail:
248 if (ret > 0)
249 ret = -ENOENT;
250 return ERR_PTR(error: ret);
251}
252
253int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans,
254 struct btrfs_root *root,
255 struct btrfs_path *path, u64 objectid,
256 u64 offset, int mod)
257{
258 struct btrfs_key file_key;
259 int ins_len = mod < 0 ? -1 : 0;
260 int cow = mod != 0;
261
262 file_key.objectid = objectid;
263 file_key.type = BTRFS_EXTENT_DATA_KEY;
264 file_key.offset = offset;
265
266 return btrfs_search_slot(trans, root, key: &file_key, p: path, ins_len, cow);
267}
268
269/*
270 * Find checksums for logical bytenr range [disk_bytenr, disk_bytenr + len) and
271 * store the result to @dst.
272 *
273 * Return >0 for the number of sectors we found.
274 * Return 0 for the range [disk_bytenr, disk_bytenr + sectorsize) has no csum
275 * for it. Caller may want to try next sector until one range is hit.
276 * Return <0 for fatal error.
277 */
278static int search_csum_tree(struct btrfs_fs_info *fs_info,
279 struct btrfs_path *path, u64 disk_bytenr,
280 u64 len, u8 *dst)
281{
282 struct btrfs_root *csum_root;
283 struct btrfs_csum_item *item = NULL;
284 struct btrfs_key key;
285 const u32 sectorsize = fs_info->sectorsize;
286 const u32 csum_size = fs_info->csum_size;
287 u32 itemsize;
288 int ret;
289 u64 csum_start;
290 u64 csum_len;
291
292 ASSERT(IS_ALIGNED(disk_bytenr, sectorsize) &&
293 IS_ALIGNED(len, sectorsize));
294
295 /* Check if the current csum item covers disk_bytenr */
296 if (path->nodes[0]) {
297 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
298 struct btrfs_csum_item);
299 btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &key, nr: path->slots[0]);
300 itemsize = btrfs_item_size(eb: path->nodes[0], slot: path->slots[0]);
301
302 csum_start = key.offset;
303 csum_len = (itemsize / csum_size) * sectorsize;
304
305 if (in_range(disk_bytenr, csum_start, csum_len))
306 goto found;
307 }
308
309 /* Current item doesn't contain the desired range, search again */
310 btrfs_release_path(p: path);
311 csum_root = btrfs_csum_root(fs_info, bytenr: disk_bytenr);
312 item = btrfs_lookup_csum(NULL, root: csum_root, path, bytenr: disk_bytenr, cow: 0);
313 if (IS_ERR(ptr: item)) {
314 ret = PTR_ERR(ptr: item);
315 goto out;
316 }
317 btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &key, nr: path->slots[0]);
318 itemsize = btrfs_item_size(eb: path->nodes[0], slot: path->slots[0]);
319
320 csum_start = key.offset;
321 csum_len = (itemsize / csum_size) * sectorsize;
322 ASSERT(in_range(disk_bytenr, csum_start, csum_len));
323
324found:
325 ret = (min(csum_start + csum_len, disk_bytenr + len) -
326 disk_bytenr) >> fs_info->sectorsize_bits;
327 read_extent_buffer(eb: path->nodes[0], dst, start: (unsigned long)item,
328 len: ret * csum_size);
329out:
330 if (ret == -ENOENT || ret == -EFBIG)
331 ret = 0;
332 return ret;
333}
334
335/*
336 * Lookup the checksum for the read bio in csum tree.
337 *
338 * Return: BLK_STS_RESOURCE if allocating memory fails, BLK_STS_OK otherwise.
339 */
340int btrfs_lookup_bio_sums(struct btrfs_bio *bbio)
341{
342 struct btrfs_inode *inode = bbio->inode;
343 struct btrfs_fs_info *fs_info = inode->root->fs_info;
344 struct bio *bio = &bbio->bio;
345 BTRFS_PATH_AUTO_FREE(path);
346 const u32 sectorsize = fs_info->sectorsize;
347 const u32 csum_size = fs_info->csum_size;
348 u32 orig_len = bio->bi_iter.bi_size;
349 u64 orig_disk_bytenr = bio->bi_iter.bi_sector << SECTOR_SHIFT;
350 const unsigned int nblocks = orig_len >> fs_info->sectorsize_bits;
351 int ret = 0;
352 u32 bio_offset = 0;
353
354 if ((inode->flags & BTRFS_INODE_NODATASUM) ||
355 test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state))
356 return 0;
357
358 /*
359 * This function is only called for read bio.
360 *
361 * This means two things:
362 * - All our csums should only be in csum tree
363 * No ordered extents csums, as ordered extents are only for write
364 * path.
365 * - No need to bother any other info from bvec
366 * Since we're looking up csums, the only important info is the
367 * disk_bytenr and the length, which can be extracted from bi_iter
368 * directly.
369 */
370 ASSERT(bio_op(bio) == REQ_OP_READ);
371 path = btrfs_alloc_path();
372 if (!path)
373 return -ENOMEM;
374
375 if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) {
376 bbio->csum = kvcalloc(nblocks, csum_size, GFP_NOFS);
377 if (!bbio->csum)
378 return -ENOMEM;
379 } else {
380 bbio->csum = bbio->csum_inline;
381 }
382
383 /*
384 * If requested number of sectors is larger than one leaf can contain,
385 * kick the readahead for csum tree.
386 */
387 if (nblocks > fs_info->csums_per_leaf)
388 path->reada = READA_FORWARD;
389
390 /*
391 * the free space stuff is only read when it hasn't been
392 * updated in the current transaction. So, we can safely
393 * read from the commit root and sidestep a nasty deadlock
394 * between reading the free space cache and updating the csum tree.
395 */
396 if (btrfs_is_free_space_inode(inode)) {
397 path->search_commit_root = true;
398 path->skip_locking = true;
399 }
400
401 /*
402 * If we are searching for a csum of an extent from a past
403 * transaction, we can search in the commit root and reduce
404 * lock contention on the csum tree extent buffers.
405 *
406 * This is important because that lock is an rwsem which gets
407 * pretty heavy write load under memory pressure and sustained
408 * csum overwrites, unlike the commit_root_sem. (Memory pressure
409 * makes us writeback the nodes multiple times per transaction,
410 * which makes us cow them each time, taking the write lock.)
411 *
412 * Due to how rwsem is implemented, there is a possible
413 * priority inversion where the readers holding the lock don't
414 * get scheduled (say they're in a cgroup stuck in heavy reclaim)
415 * which then blocks writers, including transaction commit. By
416 * using a semaphore with fewer writers (only a commit switching
417 * the roots), we make this issue less likely.
418 *
419 * Note that we don't rely on btrfs_search_slot to lock the
420 * commit root csum. We call search_slot multiple times, which would
421 * create a potential race where a commit comes in between searches
422 * while we are not holding the commit_root_sem, and we get csums
423 * from across transactions.
424 */
425 if (bbio->csum_search_commit_root) {
426 path->search_commit_root = true;
427 path->skip_locking = true;
428 down_read(sem: &fs_info->commit_root_sem);
429 }
430
431 while (bio_offset < orig_len) {
432 int count;
433 u64 cur_disk_bytenr = orig_disk_bytenr + bio_offset;
434 u8 *csum_dst = bbio->csum +
435 (bio_offset >> fs_info->sectorsize_bits) * csum_size;
436
437 count = search_csum_tree(fs_info, path, disk_bytenr: cur_disk_bytenr,
438 len: orig_len - bio_offset, dst: csum_dst);
439 if (count < 0) {
440 ret = count;
441 if (bbio->csum != bbio->csum_inline)
442 kvfree(addr: bbio->csum);
443 bbio->csum = NULL;
444 break;
445 }
446
447 /*
448 * We didn't find a csum for this range. We need to make sure
449 * we complain loudly about this, because we are not NODATASUM.
450 *
451 * However for the DATA_RELOC inode we could potentially be
452 * relocating data extents for a NODATASUM inode, so the inode
453 * itself won't be marked with NODATASUM, but the extent we're
454 * copying is in fact NODATASUM. If we don't find a csum we
455 * assume this is the case.
456 */
457 if (count == 0) {
458 memset(csum_dst, 0, csum_size);
459 count = 1;
460
461 if (btrfs_is_data_reloc_root(root: inode->root)) {
462 u64 file_offset = bbio->file_offset + bio_offset;
463
464 btrfs_set_extent_bit(tree: &inode->io_tree, start: file_offset,
465 end: file_offset + sectorsize - 1,
466 bits: EXTENT_NODATASUM, NULL);
467 } else {
468 btrfs_warn_rl(fs_info,
469 "csum hole found for disk bytenr range [%llu, %llu)",
470 cur_disk_bytenr, cur_disk_bytenr + sectorsize);
471 }
472 }
473 bio_offset += count * sectorsize;
474 }
475
476 if (bbio->csum_search_commit_root)
477 up_read(sem: &fs_info->commit_root_sem);
478 return ret;
479}
480
481/*
482 * Search for checksums for a given logical range.
483 *
484 * @root: The root where to look for checksums.
485 * @start: Logical address of target checksum range.
486 * @end: End offset (inclusive) of the target checksum range.
487 * @list: List for adding each checksum that was found.
488 * Can be NULL in case the caller only wants to check if
489 * there any checksums for the range.
490 * @nowait: Indicate if the search must be non-blocking or not.
491 *
492 * Return < 0 on error, 0 if no checksums were found, or 1 if checksums were
493 * found.
494 */
495int btrfs_lookup_csums_list(struct btrfs_root *root, u64 start, u64 end,
496 struct list_head *list, bool nowait)
497{
498 struct btrfs_fs_info *fs_info = root->fs_info;
499 struct btrfs_key key;
500 struct btrfs_path *path;
501 struct extent_buffer *leaf;
502 struct btrfs_ordered_sum *sums;
503 struct btrfs_csum_item *item;
504 int ret;
505 bool found_csums = false;
506
507 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
508 IS_ALIGNED(end + 1, fs_info->sectorsize));
509
510 path = btrfs_alloc_path();
511 if (!path)
512 return -ENOMEM;
513
514 path->nowait = nowait;
515
516 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
517 key.type = BTRFS_EXTENT_CSUM_KEY;
518 key.offset = start;
519
520 ret = btrfs_search_slot(NULL, root, key: &key, p: path, ins_len: 0, cow: 0);
521 if (ret < 0)
522 goto out;
523 if (ret > 0 && path->slots[0] > 0) {
524 leaf = path->nodes[0];
525 btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0] - 1);
526
527 /*
528 * There are two cases we can hit here for the previous csum
529 * item:
530 *
531 * |<- search range ->|
532 * |<- csum item ->|
533 *
534 * Or
535 * |<- search range ->|
536 * |<- csum item ->|
537 *
538 * Check if the previous csum item covers the leading part of
539 * the search range. If so we have to start from previous csum
540 * item.
541 */
542 if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID &&
543 key.type == BTRFS_EXTENT_CSUM_KEY) {
544 if (bytes_to_csum_size(fs_info, bytes: start - key.offset) <
545 btrfs_item_size(eb: leaf, slot: path->slots[0] - 1))
546 path->slots[0]--;
547 }
548 }
549
550 while (start <= end) {
551 u64 csum_end;
552
553 leaf = path->nodes[0];
554 if (path->slots[0] >= btrfs_header_nritems(eb: leaf)) {
555 ret = btrfs_next_leaf(root, path);
556 if (ret < 0)
557 goto out;
558 if (ret > 0)
559 break;
560 leaf = path->nodes[0];
561 }
562
563 btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
564 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
565 key.type != BTRFS_EXTENT_CSUM_KEY ||
566 key.offset > end)
567 break;
568
569 if (key.offset > start)
570 start = key.offset;
571
572 csum_end = key.offset + csum_size_to_bytes(fs_info,
573 csum_size: btrfs_item_size(eb: leaf, slot: path->slots[0]));
574 if (csum_end <= start) {
575 path->slots[0]++;
576 continue;
577 }
578
579 found_csums = true;
580 if (!list)
581 goto out;
582
583 csum_end = min(csum_end, end + 1);
584 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
585 struct btrfs_csum_item);
586 while (start < csum_end) {
587 unsigned long offset;
588 size_t size;
589
590 size = min_t(size_t, csum_end - start,
591 max_ordered_sum_bytes(fs_info));
592 sums = kzalloc(btrfs_ordered_sum_size(fs_info, size),
593 GFP_NOFS);
594 if (!sums) {
595 ret = -ENOMEM;
596 goto out;
597 }
598
599 sums->logical = start;
600 sums->len = size;
601
602 offset = bytes_to_csum_size(fs_info, bytes: start - key.offset);
603
604 read_extent_buffer(eb: path->nodes[0],
605 dst: sums->sums,
606 start: ((unsigned long)item) + offset,
607 len: bytes_to_csum_size(fs_info, bytes: size));
608
609 start += size;
610 list_add_tail(new: &sums->list, head: list);
611 }
612 path->slots[0]++;
613 }
614out:
615 btrfs_free_path(p: path);
616 if (ret < 0) {
617 if (list) {
618 struct btrfs_ordered_sum *tmp_sums;
619
620 list_for_each_entry_safe(sums, tmp_sums, list, list)
621 kfree(objp: sums);
622 }
623
624 return ret;
625 }
626
627 return found_csums ? 1 : 0;
628}
629
630/*
631 * Do the same work as btrfs_lookup_csums_list(), the difference is in how
632 * we return the result.
633 *
634 * This version will set the corresponding bits in @csum_bitmap to represent
635 * that there is a csum found.
636 * Each bit represents a sector. Thus caller should ensure @csum_buf passed
637 * in is large enough to contain all csums.
638 */
639int btrfs_lookup_csums_bitmap(struct btrfs_root *root, struct btrfs_path *path,
640 u64 start, u64 end, u8 *csum_buf,
641 unsigned long *csum_bitmap)
642{
643 struct btrfs_fs_info *fs_info = root->fs_info;
644 struct btrfs_key key;
645 struct extent_buffer *leaf;
646 struct btrfs_csum_item *item;
647 const u64 orig_start = start;
648 bool free_path = false;
649 int ret;
650
651 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
652 IS_ALIGNED(end + 1, fs_info->sectorsize));
653
654 if (!path) {
655 path = btrfs_alloc_path();
656 if (!path)
657 return -ENOMEM;
658 free_path = true;
659 }
660
661 /* Check if we can reuse the previous path. */
662 if (path->nodes[0]) {
663 btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &key, nr: path->slots[0]);
664
665 if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID &&
666 key.type == BTRFS_EXTENT_CSUM_KEY &&
667 key.offset <= start)
668 goto search_forward;
669 btrfs_release_path(p: path);
670 }
671
672 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
673 key.type = BTRFS_EXTENT_CSUM_KEY;
674 key.offset = start;
675
676 ret = btrfs_search_slot(NULL, root, key: &key, p: path, ins_len: 0, cow: 0);
677 if (ret < 0)
678 goto fail;
679 if (ret > 0 && path->slots[0] > 0) {
680 leaf = path->nodes[0];
681 btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0] - 1);
682
683 /*
684 * There are two cases we can hit here for the previous csum
685 * item:
686 *
687 * |<- search range ->|
688 * |<- csum item ->|
689 *
690 * Or
691 * |<- search range ->|
692 * |<- csum item ->|
693 *
694 * Check if the previous csum item covers the leading part of
695 * the search range. If so we have to start from previous csum
696 * item.
697 */
698 if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID &&
699 key.type == BTRFS_EXTENT_CSUM_KEY) {
700 if (bytes_to_csum_size(fs_info, bytes: start - key.offset) <
701 btrfs_item_size(eb: leaf, slot: path->slots[0] - 1))
702 path->slots[0]--;
703 }
704 }
705
706search_forward:
707 while (start <= end) {
708 u64 csum_end;
709
710 leaf = path->nodes[0];
711 if (path->slots[0] >= btrfs_header_nritems(eb: leaf)) {
712 ret = btrfs_next_leaf(root, path);
713 if (ret < 0)
714 goto fail;
715 if (ret > 0)
716 break;
717 leaf = path->nodes[0];
718 }
719
720 btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
721 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
722 key.type != BTRFS_EXTENT_CSUM_KEY ||
723 key.offset > end)
724 break;
725
726 if (key.offset > start)
727 start = key.offset;
728
729 csum_end = key.offset + csum_size_to_bytes(fs_info,
730 csum_size: btrfs_item_size(eb: leaf, slot: path->slots[0]));
731 if (csum_end <= start) {
732 path->slots[0]++;
733 continue;
734 }
735
736 csum_end = min(csum_end, end + 1);
737 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
738 struct btrfs_csum_item);
739 while (start < csum_end) {
740 unsigned long offset;
741 size_t size;
742 u8 *csum_dest = csum_buf + bytes_to_csum_size(fs_info,
743 bytes: start - orig_start);
744
745 size = min_t(size_t, csum_end - start, end + 1 - start);
746
747 offset = bytes_to_csum_size(fs_info, bytes: start - key.offset);
748
749 read_extent_buffer(eb: path->nodes[0], dst: csum_dest,
750 start: ((unsigned long)item) + offset,
751 len: bytes_to_csum_size(fs_info, bytes: size));
752
753 bitmap_set(map: csum_bitmap,
754 start: (start - orig_start) >> fs_info->sectorsize_bits,
755 nbits: size >> fs_info->sectorsize_bits);
756
757 start += size;
758 }
759 path->slots[0]++;
760 }
761 ret = 0;
762fail:
763 if (free_path)
764 btrfs_free_path(p: path);
765 return ret;
766}
767
768static void csum_one_bio(struct btrfs_bio *bbio, struct bvec_iter *src)
769{
770 struct btrfs_inode *inode = bbio->inode;
771 struct btrfs_fs_info *fs_info = inode->root->fs_info;
772 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
773 struct bio *bio = &bbio->bio;
774 struct btrfs_ordered_sum *sums = bbio->sums;
775 struct bvec_iter iter = *src;
776 phys_addr_t paddr;
777 const u32 blocksize = fs_info->sectorsize;
778 const u32 step = min(blocksize, PAGE_SIZE);
779 const u32 nr_steps = blocksize / step;
780 phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE];
781 u32 offset = 0;
782 int index = 0;
783
784 shash->tfm = fs_info->csum_shash;
785
786 btrfs_bio_for_each_block(paddr, bio, &iter, step) {
787 paddrs[(offset / step) % nr_steps] = paddr;
788 offset += step;
789
790 if (IS_ALIGNED(offset, blocksize)) {
791 btrfs_calculate_block_csum_pages(fs_info, paddrs, dest: sums->sums + index);
792 index += fs_info->csum_size;
793 }
794 }
795}
796
797static void csum_one_bio_work(struct work_struct *work)
798{
799 struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, csum_work);
800
801 ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE);
802 ASSERT(bbio->async_csum == true);
803 csum_one_bio(bbio, src: &bbio->csum_saved_iter);
804 complete(&bbio->csum_done);
805}
806
807/*
808 * Calculate checksums of the data contained inside a bio.
809 */
810int btrfs_csum_one_bio(struct btrfs_bio *bbio, bool async)
811{
812 struct btrfs_ordered_extent *ordered = bbio->ordered;
813 struct btrfs_inode *inode = bbio->inode;
814 struct btrfs_fs_info *fs_info = inode->root->fs_info;
815 struct bio *bio = &bbio->bio;
816 struct btrfs_ordered_sum *sums;
817 unsigned nofs_flag;
818
819 nofs_flag = memalloc_nofs_save();
820 sums = kvzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size),
821 GFP_KERNEL);
822 memalloc_nofs_restore(flags: nofs_flag);
823
824 if (!sums)
825 return -ENOMEM;
826
827 sums->logical = bbio->orig_logical;
828 sums->len = bio->bi_iter.bi_size;
829 INIT_LIST_HEAD(list: &sums->list);
830 bbio->sums = sums;
831 btrfs_add_ordered_sum(entry: ordered, sum: sums);
832
833 if (!async) {
834 csum_one_bio(bbio, src: &bbio->bio.bi_iter);
835 return 0;
836 }
837 init_completion(x: &bbio->csum_done);
838 bbio->async_csum = true;
839 bbio->csum_saved_iter = bbio->bio.bi_iter;
840 INIT_WORK(&bbio->csum_work, csum_one_bio_work);
841 schedule_work(work: &bbio->csum_work);
842 return 0;
843}
844
845/*
846 * Nodatasum I/O on zoned file systems still requires an btrfs_ordered_sum to
847 * record the updated logical address on Zone Append completion.
848 * Allocate just the structure with an empty sums array here for that case.
849 */
850int btrfs_alloc_dummy_sum(struct btrfs_bio *bbio)
851{
852 bbio->sums = kmalloc(sizeof(*bbio->sums), GFP_NOFS);
853 if (!bbio->sums)
854 return -ENOMEM;
855 bbio->sums->len = bbio->bio.bi_iter.bi_size;
856 bbio->sums->logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT;
857 btrfs_add_ordered_sum(entry: bbio->ordered, sum: bbio->sums);
858 return 0;
859}
860
861/*
862 * Remove one checksum overlapping a range.
863 *
864 * This expects the key to describe the csum pointed to by the path, and it
865 * expects the csum to overlap the range [bytenr, len]
866 *
867 * The csum should not be entirely contained in the range and the range should
868 * not be entirely contained in the csum.
869 *
870 * This calls btrfs_truncate_item with the correct args based on the overlap,
871 * and fixes up the key as required.
872 */
873static noinline void truncate_one_csum(struct btrfs_trans_handle *trans,
874 struct btrfs_path *path,
875 struct btrfs_key *key,
876 u64 bytenr, u64 len)
877{
878 struct btrfs_fs_info *fs_info = trans->fs_info;
879 struct extent_buffer *leaf;
880 const u32 csum_size = fs_info->csum_size;
881 u64 csum_end;
882 u64 end_byte = bytenr + len;
883 u32 blocksize_bits = fs_info->sectorsize_bits;
884
885 leaf = path->nodes[0];
886 csum_end = btrfs_item_size(eb: leaf, slot: path->slots[0]) / csum_size;
887 csum_end <<= blocksize_bits;
888 csum_end += key->offset;
889
890 if (key->offset < bytenr && csum_end <= end_byte) {
891 /*
892 * [ bytenr - len ]
893 * [ ]
894 * [csum ]
895 * A simple truncate off the end of the item
896 */
897 u32 new_size = (bytenr - key->offset) >> blocksize_bits;
898 new_size *= csum_size;
899 btrfs_truncate_item(trans, path, new_size, from_end: 1);
900 } else if (key->offset >= bytenr && csum_end > end_byte &&
901 end_byte > key->offset) {
902 /*
903 * [ bytenr - len ]
904 * [ ]
905 * [csum ]
906 * we need to truncate from the beginning of the csum
907 */
908 u32 new_size = (csum_end - end_byte) >> blocksize_bits;
909 new_size *= csum_size;
910
911 btrfs_truncate_item(trans, path, new_size, from_end: 0);
912
913 key->offset = end_byte;
914 btrfs_set_item_key_safe(trans, path, new_key: key);
915 } else {
916 BUG();
917 }
918}
919
920/*
921 * Delete the csum items from the csum tree for a given range of bytes.
922 */
923int btrfs_del_csums(struct btrfs_trans_handle *trans,
924 struct btrfs_root *root, u64 bytenr, u64 len)
925{
926 struct btrfs_fs_info *fs_info = trans->fs_info;
927 BTRFS_PATH_AUTO_FREE(path);
928 struct btrfs_key key;
929 u64 end_byte = bytenr + len;
930 u64 csum_end;
931 struct extent_buffer *leaf;
932 int ret = 0;
933 const u32 csum_size = fs_info->csum_size;
934 u32 blocksize_bits = fs_info->sectorsize_bits;
935
936 ASSERT(btrfs_root_id(root) == BTRFS_CSUM_TREE_OBJECTID ||
937 btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID);
938
939 path = btrfs_alloc_path();
940 if (!path)
941 return -ENOMEM;
942
943 while (1) {
944 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
945 key.type = BTRFS_EXTENT_CSUM_KEY;
946 key.offset = end_byte - 1;
947
948 ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: -1, cow: 1);
949 if (ret > 0) {
950 ret = 0;
951 if (path->slots[0] == 0)
952 break;
953 path->slots[0]--;
954 } else if (ret < 0) {
955 break;
956 }
957
958 leaf = path->nodes[0];
959 btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
960
961 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
962 key.type != BTRFS_EXTENT_CSUM_KEY) {
963 break;
964 }
965
966 if (key.offset >= end_byte)
967 break;
968
969 csum_end = btrfs_item_size(eb: leaf, slot: path->slots[0]) / csum_size;
970 csum_end <<= blocksize_bits;
971 csum_end += key.offset;
972
973 /* this csum ends before we start, we're done */
974 if (csum_end <= bytenr)
975 break;
976
977 /* delete the entire item, it is inside our range */
978 if (key.offset >= bytenr && csum_end <= end_byte) {
979 int del_nr = 1;
980
981 /*
982 * Check how many csum items preceding this one in this
983 * leaf correspond to our range and then delete them all
984 * at once.
985 */
986 if (key.offset > bytenr && path->slots[0] > 0) {
987 int slot = path->slots[0] - 1;
988
989 while (slot >= 0) {
990 struct btrfs_key pk;
991
992 btrfs_item_key_to_cpu(eb: leaf, cpu_key: &pk, nr: slot);
993 if (pk.offset < bytenr ||
994 pk.type != BTRFS_EXTENT_CSUM_KEY ||
995 pk.objectid !=
996 BTRFS_EXTENT_CSUM_OBJECTID)
997 break;
998 path->slots[0] = slot;
999 del_nr++;
1000 key.offset = pk.offset;
1001 slot--;
1002 }
1003 }
1004 ret = btrfs_del_items(trans, root, path,
1005 slot: path->slots[0], nr: del_nr);
1006 if (ret)
1007 break;
1008 if (key.offset == bytenr)
1009 break;
1010 } else if (key.offset < bytenr && csum_end > end_byte) {
1011 unsigned long offset;
1012 unsigned long shift_len;
1013 unsigned long item_offset;
1014 /*
1015 * [ bytenr - len ]
1016 * [csum ]
1017 *
1018 * Our bytes are in the middle of the csum,
1019 * we need to split this item and insert a new one.
1020 *
1021 * But we can't drop the path because the
1022 * csum could change, get removed, extended etc.
1023 *
1024 * The trick here is the max size of a csum item leaves
1025 * enough room in the tree block for a single
1026 * item header. So, we split the item in place,
1027 * adding a new header pointing to the existing
1028 * bytes. Then we loop around again and we have
1029 * a nicely formed csum item that we can neatly
1030 * truncate.
1031 */
1032 offset = (bytenr - key.offset) >> blocksize_bits;
1033 offset *= csum_size;
1034
1035 shift_len = (len >> blocksize_bits) * csum_size;
1036
1037 item_offset = btrfs_item_ptr_offset(leaf,
1038 path->slots[0]);
1039
1040 memzero_extent_buffer(eb: leaf, start: item_offset + offset,
1041 len: shift_len);
1042 key.offset = bytenr;
1043
1044 /*
1045 * btrfs_split_item returns -EAGAIN when the
1046 * item changed size or key
1047 */
1048 ret = btrfs_split_item(trans, root, path, new_key: &key, split_offset: offset);
1049 if (unlikely(ret && ret != -EAGAIN)) {
1050 btrfs_abort_transaction(trans, ret);
1051 break;
1052 }
1053 ret = 0;
1054
1055 key.offset = end_byte - 1;
1056 } else {
1057 truncate_one_csum(trans, path, key: &key, bytenr, len);
1058 if (key.offset < bytenr)
1059 break;
1060 }
1061 btrfs_release_path(p: path);
1062 }
1063 return ret;
1064}
1065
1066static int find_next_csum_offset(struct btrfs_root *root,
1067 struct btrfs_path *path,
1068 u64 *next_offset)
1069{
1070 const u32 nritems = btrfs_header_nritems(eb: path->nodes[0]);
1071 struct btrfs_key found_key;
1072 int slot = path->slots[0] + 1;
1073 int ret;
1074
1075 if (nritems == 0 || slot >= nritems) {
1076 ret = btrfs_next_leaf(root, path);
1077 if (ret < 0) {
1078 return ret;
1079 } else if (ret > 0) {
1080 *next_offset = (u64)-1;
1081 return 0;
1082 }
1083 slot = path->slots[0];
1084 }
1085
1086 btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &found_key, nr: slot);
1087
1088 if (found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
1089 found_key.type != BTRFS_EXTENT_CSUM_KEY)
1090 *next_offset = (u64)-1;
1091 else
1092 *next_offset = found_key.offset;
1093
1094 return 0;
1095}
1096
1097int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans,
1098 struct btrfs_root *root,
1099 struct btrfs_ordered_sum *sums)
1100{
1101 struct btrfs_fs_info *fs_info = root->fs_info;
1102 struct btrfs_key file_key;
1103 struct btrfs_key found_key;
1104 BTRFS_PATH_AUTO_FREE(path);
1105 struct btrfs_csum_item *item;
1106 struct btrfs_csum_item *item_end;
1107 struct extent_buffer *leaf = NULL;
1108 u64 next_offset;
1109 u64 total_bytes = 0;
1110 u64 csum_offset;
1111 u64 bytenr;
1112 u32 ins_size;
1113 int index = 0;
1114 int found_next;
1115 int ret;
1116 const u32 csum_size = fs_info->csum_size;
1117
1118 path = btrfs_alloc_path();
1119 if (!path)
1120 return -ENOMEM;
1121again:
1122 next_offset = (u64)-1;
1123 found_next = 0;
1124 bytenr = sums->logical + total_bytes;
1125 file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
1126 file_key.type = BTRFS_EXTENT_CSUM_KEY;
1127 file_key.offset = bytenr;
1128
1129 item = btrfs_lookup_csum(trans, root, path, bytenr, cow: 1);
1130 if (!IS_ERR(ptr: item)) {
1131 ret = 0;
1132 leaf = path->nodes[0];
1133 item_end = btrfs_item_ptr(leaf, path->slots[0],
1134 struct btrfs_csum_item);
1135 item_end = (struct btrfs_csum_item *)((char *)item_end +
1136 btrfs_item_size(eb: leaf, slot: path->slots[0]));
1137 goto found;
1138 }
1139 ret = PTR_ERR(ptr: item);
1140 if (ret != -EFBIG && ret != -ENOENT)
1141 goto out;
1142
1143 if (ret == -EFBIG) {
1144 u32 item_size;
1145 /* we found one, but it isn't big enough yet */
1146 leaf = path->nodes[0];
1147 item_size = btrfs_item_size(eb: leaf, slot: path->slots[0]);
1148 if ((item_size / csum_size) >=
1149 MAX_CSUM_ITEMS(fs_info, csum_size)) {
1150 /* already at max size, make a new one */
1151 goto insert;
1152 }
1153 } else {
1154 /* We didn't find a csum item, insert one. */
1155 ret = find_next_csum_offset(root, path, next_offset: &next_offset);
1156 if (ret < 0)
1157 goto out;
1158 found_next = 1;
1159 goto insert;
1160 }
1161
1162 /*
1163 * At this point, we know the tree has a checksum item that ends at an
1164 * offset matching the start of the checksum range we want to insert.
1165 * We try to extend that item as much as possible and then add as many
1166 * checksums to it as they fit.
1167 *
1168 * First check if the leaf has enough free space for at least one
1169 * checksum. If it has go directly to the item extension code, otherwise
1170 * release the path and do a search for insertion before the extension.
1171 */
1172 if (btrfs_leaf_free_space(leaf) >= csum_size) {
1173 btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
1174 csum_offset = (bytenr - found_key.offset) >>
1175 fs_info->sectorsize_bits;
1176 goto extend_csum;
1177 }
1178
1179 btrfs_release_path(p: path);
1180 path->search_for_extension = true;
1181 ret = btrfs_search_slot(trans, root, key: &file_key, p: path,
1182 ins_len: csum_size, cow: 1);
1183 path->search_for_extension = false;
1184 if (ret < 0)
1185 goto out;
1186
1187 if (ret > 0) {
1188 if (path->slots[0] == 0)
1189 goto insert;
1190 path->slots[0]--;
1191 }
1192
1193 leaf = path->nodes[0];
1194 btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
1195 csum_offset = (bytenr - found_key.offset) >> fs_info->sectorsize_bits;
1196
1197 if (found_key.type != BTRFS_EXTENT_CSUM_KEY ||
1198 found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
1199 csum_offset >= MAX_CSUM_ITEMS(fs_info, csum_size)) {
1200 goto insert;
1201 }
1202
1203extend_csum:
1204 if (csum_offset == btrfs_item_size(eb: leaf, slot: path->slots[0]) /
1205 csum_size) {
1206 int extend_nr;
1207 u64 tmp;
1208 u32 diff;
1209
1210 tmp = sums->len - total_bytes;
1211 tmp >>= fs_info->sectorsize_bits;
1212 WARN_ON(tmp < 1);
1213 extend_nr = max_t(int, 1, tmp);
1214
1215 /*
1216 * A log tree can already have checksum items with a subset of
1217 * the checksums we are trying to log. This can happen after
1218 * doing a sequence of partial writes into prealloc extents and
1219 * fsyncs in between, with a full fsync logging a larger subrange
1220 * of an extent for which a previous fast fsync logged a smaller
1221 * subrange. And this happens in particular due to merging file
1222 * extent items when we complete an ordered extent for a range
1223 * covered by a prealloc extent - this is done at
1224 * btrfs_mark_extent_written().
1225 *
1226 * So if we try to extend the previous checksum item, which has
1227 * a range that ends at the start of the range we want to insert,
1228 * make sure we don't extend beyond the start offset of the next
1229 * checksum item. If we are at the last item in the leaf, then
1230 * forget the optimization of extending and add a new checksum
1231 * item - it is not worth the complexity of releasing the path,
1232 * getting the first key for the next leaf, repeat the btree
1233 * search, etc, because log trees are temporary anyway and it
1234 * would only save a few bytes of leaf space.
1235 */
1236 if (btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID) {
1237 if (path->slots[0] + 1 >=
1238 btrfs_header_nritems(eb: path->nodes[0])) {
1239 ret = find_next_csum_offset(root, path, next_offset: &next_offset);
1240 if (ret < 0)
1241 goto out;
1242 found_next = 1;
1243 goto insert;
1244 }
1245
1246 ret = find_next_csum_offset(root, path, next_offset: &next_offset);
1247 if (ret < 0)
1248 goto out;
1249
1250 tmp = (next_offset - bytenr) >> fs_info->sectorsize_bits;
1251 if (tmp <= INT_MAX)
1252 extend_nr = min_t(int, extend_nr, tmp);
1253 }
1254
1255 diff = (csum_offset + extend_nr) * csum_size;
1256 diff = min(diff,
1257 MAX_CSUM_ITEMS(fs_info, csum_size) * csum_size);
1258
1259 diff = diff - btrfs_item_size(eb: leaf, slot: path->slots[0]);
1260 diff = min_t(u32, btrfs_leaf_free_space(leaf), diff);
1261 diff /= csum_size;
1262 diff *= csum_size;
1263
1264 btrfs_extend_item(trans, path, data_size: diff);
1265 ret = 0;
1266 goto csum;
1267 }
1268
1269insert:
1270 btrfs_release_path(p: path);
1271 csum_offset = 0;
1272 if (found_next) {
1273 u64 tmp;
1274
1275 tmp = sums->len - total_bytes;
1276 tmp >>= fs_info->sectorsize_bits;
1277 tmp = min(tmp, (next_offset - file_key.offset) >>
1278 fs_info->sectorsize_bits);
1279
1280 tmp = max_t(u64, 1, tmp);
1281 tmp = min_t(u64, tmp, MAX_CSUM_ITEMS(fs_info, csum_size));
1282 ins_size = csum_size * tmp;
1283 } else {
1284 ins_size = csum_size;
1285 }
1286 ret = btrfs_insert_empty_item(trans, root, path, key: &file_key,
1287 data_size: ins_size);
1288 if (ret < 0)
1289 goto out;
1290 leaf = path->nodes[0];
1291csum:
1292 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);
1293 item_end = (struct btrfs_csum_item *)((unsigned char *)item +
1294 btrfs_item_size(eb: leaf, slot: path->slots[0]));
1295 item = (struct btrfs_csum_item *)((unsigned char *)item +
1296 csum_offset * csum_size);
1297found:
1298 ins_size = (u32)(sums->len - total_bytes) >> fs_info->sectorsize_bits;
1299 ins_size *= csum_size;
1300 ins_size = min_t(u32, (unsigned long)item_end - (unsigned long)item,
1301 ins_size);
1302 write_extent_buffer(eb: leaf, src: sums->sums + index, start: (unsigned long)item,
1303 len: ins_size);
1304
1305 index += ins_size;
1306 ins_size /= csum_size;
1307 total_bytes += ins_size * fs_info->sectorsize;
1308
1309 if (total_bytes < sums->len) {
1310 btrfs_release_path(p: path);
1311 cond_resched();
1312 goto again;
1313 }
1314out:
1315 return ret;
1316}
1317
1318void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode,
1319 const struct btrfs_path *path,
1320 const struct btrfs_file_extent_item *fi,
1321 struct extent_map *em)
1322{
1323 struct btrfs_fs_info *fs_info = inode->root->fs_info;
1324 struct btrfs_root *root = inode->root;
1325 struct extent_buffer *leaf = path->nodes[0];
1326 const int slot = path->slots[0];
1327 struct btrfs_key key;
1328 u64 extent_start;
1329 u8 type = btrfs_file_extent_type(eb: leaf, s: fi);
1330 int compress_type = btrfs_file_extent_compression(eb: leaf, s: fi);
1331
1332 btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: slot);
1333 extent_start = key.offset;
1334 em->ram_bytes = btrfs_file_extent_ram_bytes(eb: leaf, s: fi);
1335 em->generation = btrfs_file_extent_generation(eb: leaf, s: fi);
1336 if (type == BTRFS_FILE_EXTENT_REG ||
1337 type == BTRFS_FILE_EXTENT_PREALLOC) {
1338 const u64 disk_bytenr = btrfs_file_extent_disk_bytenr(eb: leaf, s: fi);
1339
1340 em->start = extent_start;
1341 em->len = btrfs_file_extent_end(path) - extent_start;
1342 if (disk_bytenr == 0) {
1343 em->disk_bytenr = EXTENT_MAP_HOLE;
1344 em->disk_num_bytes = 0;
1345 em->offset = 0;
1346 return;
1347 }
1348 em->disk_bytenr = disk_bytenr;
1349 em->disk_num_bytes = btrfs_file_extent_disk_num_bytes(eb: leaf, s: fi);
1350 em->offset = btrfs_file_extent_offset(eb: leaf, s: fi);
1351 if (compress_type != BTRFS_COMPRESS_NONE) {
1352 btrfs_extent_map_set_compression(em, type: compress_type);
1353 } else {
1354 /*
1355 * Older kernels can create regular non-hole data
1356 * extents with ram_bytes smaller than disk_num_bytes.
1357 * Not a big deal, just always use disk_num_bytes
1358 * for ram_bytes.
1359 */
1360 em->ram_bytes = em->disk_num_bytes;
1361 if (type == BTRFS_FILE_EXTENT_PREALLOC)
1362 em->flags |= EXTENT_FLAG_PREALLOC;
1363 }
1364 } else if (type == BTRFS_FILE_EXTENT_INLINE) {
1365 /* Tree-checker has ensured this. */
1366 ASSERT(extent_start == 0);
1367
1368 em->disk_bytenr = EXTENT_MAP_INLINE;
1369 em->start = 0;
1370 em->len = fs_info->sectorsize;
1371 em->offset = 0;
1372 btrfs_extent_map_set_compression(em, type: compress_type);
1373 } else {
1374 btrfs_err(fs_info,
1375 "unknown file extent item type %d, inode %llu, offset %llu, "
1376 "root %llu", type, btrfs_ino(inode), extent_start,
1377 btrfs_root_id(root));
1378 }
1379}
1380
1381/*
1382 * Returns the end offset (non inclusive) of the file extent item the given path
1383 * points to. If it points to an inline extent, the returned offset is rounded
1384 * up to the sector size.
1385 */
1386u64 btrfs_file_extent_end(const struct btrfs_path *path)
1387{
1388 const struct extent_buffer *leaf = path->nodes[0];
1389 const int slot = path->slots[0];
1390 struct btrfs_file_extent_item *fi;
1391 struct btrfs_key key;
1392 u64 end;
1393
1394 btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: slot);
1395 ASSERT(key.type == BTRFS_EXTENT_DATA_KEY);
1396 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
1397
1398 if (btrfs_file_extent_type(eb: leaf, s: fi) == BTRFS_FILE_EXTENT_INLINE)
1399 end = leaf->fs_info->sectorsize;
1400 else
1401 end = key.offset + btrfs_file_extent_num_bytes(eb: leaf, s: fi);
1402
1403 return end;
1404}
1405

source code of linux/fs/btrfs/file-item.c