1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
4 * Written by Alex Tomas <alex@clusterfs.com>
5 *
6 * Architecture independence:
7 * Copyright (c) 2005, Bull S.A.
8 * Written by Pierre Peiffer <pierre.peiffer@bull.net>
9 */
10
11/*
12 * Extents support for EXT4
13 *
14 * TODO:
15 * - ext4*_error() should be used in some situations
16 * - analyze all BUG()/BUG_ON(), use -EIO where appropriate
17 * - smart tree reduction
18 */
19
20#include <linux/fs.h>
21#include <linux/time.h>
22#include <linux/jbd2.h>
23#include <linux/highuid.h>
24#include <linux/pagemap.h>
25#include <linux/quotaops.h>
26#include <linux/string.h>
27#include <linux/slab.h>
28#include <linux/uaccess.h>
29#include <linux/fiemap.h>
30#include <linux/iomap.h>
31#include <linux/sched/mm.h>
32#include "ext4_jbd2.h"
33#include "ext4_extents.h"
34#include "xattr.h"
35
36#include <trace/events/ext4.h>
37
38/*
39 * used by extent splitting.
40 */
41#define EXT4_EXT_MAY_ZEROOUT 0x1 /* safe to zeroout if split fails \
42 due to ENOSPC */
43#define EXT4_EXT_MARK_UNWRIT1 0x2 /* mark first half unwritten */
44#define EXT4_EXT_MARK_UNWRIT2 0x4 /* mark second half unwritten */
45
46#define EXT4_EXT_DATA_VALID1 0x8 /* first half contains valid data */
47#define EXT4_EXT_DATA_VALID2 0x10 /* second half contains valid data */
48
49static __le32 ext4_extent_block_csum(struct inode *inode,
50 struct ext4_extent_header *eh)
51{
52 struct ext4_inode_info *ei = EXT4_I(inode);
53 __u32 csum;
54
55 csum = ext4_chksum(crc: ei->i_csum_seed, address: (__u8 *)eh,
56 EXT4_EXTENT_TAIL_OFFSET(eh));
57 return cpu_to_le32(csum);
58}
59
60static int ext4_extent_block_csum_verify(struct inode *inode,
61 struct ext4_extent_header *eh)
62{
63 struct ext4_extent_tail *et;
64
65 if (!ext4_has_feature_metadata_csum(sb: inode->i_sb))
66 return 1;
67
68 et = find_ext4_extent_tail(eh);
69 if (et->et_checksum != ext4_extent_block_csum(inode, eh))
70 return 0;
71 return 1;
72}
73
74static void ext4_extent_block_csum_set(struct inode *inode,
75 struct ext4_extent_header *eh)
76{
77 struct ext4_extent_tail *et;
78
79 if (!ext4_has_feature_metadata_csum(sb: inode->i_sb))
80 return;
81
82 et = find_ext4_extent_tail(eh);
83 et->et_checksum = ext4_extent_block_csum(inode, eh);
84}
85
86static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle,
87 struct inode *inode,
88 struct ext4_ext_path *path,
89 ext4_lblk_t split,
90 int split_flag, int flags);
91
92static int ext4_ext_trunc_restart_fn(struct inode *inode, int *dropped)
93{
94 /*
95 * Drop i_data_sem to avoid deadlock with ext4_map_blocks. At this
96 * moment, get_block can be called only for blocks inside i_size since
97 * page cache has been already dropped and writes are blocked by
98 * i_rwsem. So we can safely drop the i_data_sem here.
99 */
100 BUG_ON(EXT4_JOURNAL(inode) == NULL);
101 ext4_discard_preallocations(inode);
102 up_write(sem: &EXT4_I(inode)->i_data_sem);
103 *dropped = 1;
104 return 0;
105}
106
107static inline void ext4_ext_path_brelse(struct ext4_ext_path *path)
108{
109 brelse(bh: path->p_bh);
110 path->p_bh = NULL;
111}
112
113static void ext4_ext_drop_refs(struct ext4_ext_path *path)
114{
115 int depth, i;
116
117 if (IS_ERR_OR_NULL(ptr: path))
118 return;
119 depth = path->p_depth;
120 for (i = 0; i <= depth; i++, path++)
121 ext4_ext_path_brelse(path);
122}
123
124void ext4_free_ext_path(struct ext4_ext_path *path)
125{
126 if (IS_ERR_OR_NULL(ptr: path))
127 return;
128 ext4_ext_drop_refs(path);
129 kfree(objp: path);
130}
131
132/*
133 * Make sure 'handle' has at least 'check_cred' credits. If not, restart
134 * transaction with 'restart_cred' credits. The function drops i_data_sem
135 * when restarting transaction and gets it after transaction is restarted.
136 *
137 * The function returns 0 on success, 1 if transaction had to be restarted,
138 * and < 0 in case of fatal error.
139 */
140int ext4_datasem_ensure_credits(handle_t *handle, struct inode *inode,
141 int check_cred, int restart_cred,
142 int revoke_cred)
143{
144 int ret;
145 int dropped = 0;
146
147 ret = ext4_journal_ensure_credits_fn(handle, check_cred, restart_cred,
148 revoke_cred, ext4_ext_trunc_restart_fn(inode, &dropped));
149 if (dropped)
150 down_write(sem: &EXT4_I(inode)->i_data_sem);
151 return ret;
152}
153
154/*
155 * could return:
156 * - EROFS
157 * - ENOMEM
158 */
159static int ext4_ext_get_access(handle_t *handle, struct inode *inode,
160 struct ext4_ext_path *path)
161{
162 int err = 0;
163
164 if (path->p_bh) {
165 /* path points to block */
166 BUFFER_TRACE(path->p_bh, "get_write_access");
167 err = ext4_journal_get_write_access(handle, inode->i_sb,
168 path->p_bh, EXT4_JTR_NONE);
169 /*
170 * The extent buffer's verified bit will be set again in
171 * __ext4_ext_dirty(). We could leave an inconsistent
172 * buffer if the extents updating procudure break off du
173 * to some error happens, force to check it again.
174 */
175 if (!err)
176 clear_buffer_verified(bh: path->p_bh);
177 }
178 /* path points to leaf/index in inode body */
179 /* we use in-core data, no need to protect them */
180 return err;
181}
182
183/*
184 * could return:
185 * - EROFS
186 * - ENOMEM
187 * - EIO
188 */
189static int __ext4_ext_dirty(const char *where, unsigned int line,
190 handle_t *handle, struct inode *inode,
191 struct ext4_ext_path *path)
192{
193 int err;
194
195 WARN_ON(!rwsem_is_locked(&EXT4_I(inode)->i_data_sem));
196 if (path->p_bh) {
197 ext4_extent_block_csum_set(inode, eh: ext_block_hdr(bh: path->p_bh));
198 /* path points to block */
199 err = __ext4_handle_dirty_metadata(where, line, handle,
200 inode, bh: path->p_bh);
201 /* Extents updating done, re-set verified flag */
202 if (!err)
203 set_buffer_verified(path->p_bh);
204 } else {
205 /* path points to leaf/index in inode body */
206 err = ext4_mark_inode_dirty(handle, inode);
207 }
208 return err;
209}
210
211#define ext4_ext_dirty(handle, inode, path) \
212 __ext4_ext_dirty(__func__, __LINE__, (handle), (inode), (path))
213
214static ext4_fsblk_t ext4_ext_find_goal(struct inode *inode,
215 struct ext4_ext_path *path,
216 ext4_lblk_t block)
217{
218 if (path) {
219 int depth = path->p_depth;
220 struct ext4_extent *ex;
221
222 /*
223 * Try to predict block placement assuming that we are
224 * filling in a file which will eventually be
225 * non-sparse --- i.e., in the case of libbfd writing
226 * an ELF object sections out-of-order but in a way
227 * the eventually results in a contiguous object or
228 * executable file, or some database extending a table
229 * space file. However, this is actually somewhat
230 * non-ideal if we are writing a sparse file such as
231 * qemu or KVM writing a raw image file that is going
232 * to stay fairly sparse, since it will end up
233 * fragmenting the file system's free space. Maybe we
234 * should have some hueristics or some way to allow
235 * userspace to pass a hint to file system,
236 * especially if the latter case turns out to be
237 * common.
238 */
239 ex = path[depth].p_ext;
240 if (ex) {
241 ext4_fsblk_t ext_pblk = ext4_ext_pblock(ex);
242 ext4_lblk_t ext_block = le32_to_cpu(ex->ee_block);
243
244 if (block > ext_block)
245 return ext_pblk + (block - ext_block);
246 else
247 return ext_pblk - (ext_block - block);
248 }
249
250 /* it looks like index is empty;
251 * try to find starting block from index itself */
252 if (path[depth].p_bh)
253 return path[depth].p_bh->b_blocknr;
254 }
255
256 /* OK. use inode's group */
257 return ext4_inode_to_goal_block(inode);
258}
259
260/*
261 * Allocation for a meta data block
262 */
263static ext4_fsblk_t
264ext4_ext_new_meta_block(handle_t *handle, struct inode *inode,
265 struct ext4_ext_path *path,
266 struct ext4_extent *ex, int *err, unsigned int flags)
267{
268 ext4_fsblk_t goal, newblock;
269
270 goal = ext4_ext_find_goal(inode, path, le32_to_cpu(ex->ee_block));
271 newblock = ext4_new_meta_blocks(handle, inode, goal, flags,
272 NULL, errp: err);
273 return newblock;
274}
275
276static inline int ext4_ext_space_block(struct inode *inode, int check)
277{
278 int size;
279
280 size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header))
281 / sizeof(struct ext4_extent);
282#ifdef AGGRESSIVE_TEST
283 if (!check && size > 6)
284 size = 6;
285#endif
286 return size;
287}
288
289static inline int ext4_ext_space_block_idx(struct inode *inode, int check)
290{
291 int size;
292
293 size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header))
294 / sizeof(struct ext4_extent_idx);
295#ifdef AGGRESSIVE_TEST
296 if (!check && size > 5)
297 size = 5;
298#endif
299 return size;
300}
301
302static inline int ext4_ext_space_root(struct inode *inode, int check)
303{
304 int size;
305
306 size = sizeof(EXT4_I(inode)->i_data);
307 size -= sizeof(struct ext4_extent_header);
308 size /= sizeof(struct ext4_extent);
309#ifdef AGGRESSIVE_TEST
310 if (!check && size > 3)
311 size = 3;
312#endif
313 return size;
314}
315
316static inline int ext4_ext_space_root_idx(struct inode *inode, int check)
317{
318 int size;
319
320 size = sizeof(EXT4_I(inode)->i_data);
321 size -= sizeof(struct ext4_extent_header);
322 size /= sizeof(struct ext4_extent_idx);
323#ifdef AGGRESSIVE_TEST
324 if (!check && size > 4)
325 size = 4;
326#endif
327 return size;
328}
329
330static inline struct ext4_ext_path *
331ext4_force_split_extent_at(handle_t *handle, struct inode *inode,
332 struct ext4_ext_path *path, ext4_lblk_t lblk,
333 int nofail)
334{
335 int unwritten = ext4_ext_is_unwritten(ext: path[path->p_depth].p_ext);
336 int flags = EXT4_EX_NOCACHE | EXT4_GET_BLOCKS_PRE_IO;
337
338 if (nofail)
339 flags |= EXT4_GET_BLOCKS_METADATA_NOFAIL | EXT4_EX_NOFAIL;
340
341 return ext4_split_extent_at(handle, inode, path, split: lblk, split_flag: unwritten ?
342 EXT4_EXT_MARK_UNWRIT1|EXT4_EXT_MARK_UNWRIT2 : 0,
343 flags);
344}
345
346static int
347ext4_ext_max_entries(struct inode *inode, int depth)
348{
349 int max;
350
351 if (depth == ext_depth(inode)) {
352 if (depth == 0)
353 max = ext4_ext_space_root(inode, check: 1);
354 else
355 max = ext4_ext_space_root_idx(inode, check: 1);
356 } else {
357 if (depth == 0)
358 max = ext4_ext_space_block(inode, check: 1);
359 else
360 max = ext4_ext_space_block_idx(inode, check: 1);
361 }
362
363 return max;
364}
365
366static int ext4_valid_extent(struct inode *inode, struct ext4_extent *ext)
367{
368 ext4_fsblk_t block = ext4_ext_pblock(ex: ext);
369 int len = ext4_ext_get_actual_len(ext);
370 ext4_lblk_t lblock = le32_to_cpu(ext->ee_block);
371
372 /*
373 * We allow neither:
374 * - zero length
375 * - overflow/wrap-around
376 */
377 if (lblock + len <= lblock)
378 return 0;
379 return ext4_inode_block_valid(inode, start_blk: block, count: len);
380}
381
382static int ext4_valid_extent_idx(struct inode *inode,
383 struct ext4_extent_idx *ext_idx)
384{
385 ext4_fsblk_t block = ext4_idx_pblock(ix: ext_idx);
386
387 return ext4_inode_block_valid(inode, start_blk: block, count: 1);
388}
389
390static int ext4_valid_extent_entries(struct inode *inode,
391 struct ext4_extent_header *eh,
392 ext4_lblk_t lblk, ext4_fsblk_t *pblk,
393 int depth)
394{
395 unsigned short entries;
396 ext4_lblk_t lblock = 0;
397 ext4_lblk_t cur = 0;
398
399 if (eh->eh_entries == 0)
400 return 1;
401
402 entries = le16_to_cpu(eh->eh_entries);
403
404 if (depth == 0) {
405 /* leaf entries */
406 struct ext4_extent *ext = EXT_FIRST_EXTENT(eh);
407
408 /*
409 * The logical block in the first entry should equal to
410 * the number in the index block.
411 */
412 if (depth != ext_depth(inode) &&
413 lblk != le32_to_cpu(ext->ee_block))
414 return 0;
415 while (entries) {
416 if (!ext4_valid_extent(inode, ext))
417 return 0;
418
419 /* Check for overlapping extents */
420 lblock = le32_to_cpu(ext->ee_block);
421 if (lblock < cur) {
422 *pblk = ext4_ext_pblock(ex: ext);
423 return 0;
424 }
425 cur = lblock + ext4_ext_get_actual_len(ext);
426 ext++;
427 entries--;
428 }
429 } else {
430 struct ext4_extent_idx *ext_idx = EXT_FIRST_INDEX(eh);
431
432 /*
433 * The logical block in the first entry should equal to
434 * the number in the parent index block.
435 */
436 if (depth != ext_depth(inode) &&
437 lblk != le32_to_cpu(ext_idx->ei_block))
438 return 0;
439 while (entries) {
440 if (!ext4_valid_extent_idx(inode, ext_idx))
441 return 0;
442
443 /* Check for overlapping index extents */
444 lblock = le32_to_cpu(ext_idx->ei_block);
445 if (lblock < cur) {
446 *pblk = ext4_idx_pblock(ix: ext_idx);
447 return 0;
448 }
449 ext_idx++;
450 entries--;
451 cur = lblock + 1;
452 }
453 }
454 return 1;
455}
456
457static int __ext4_ext_check(const char *function, unsigned int line,
458 struct inode *inode, struct ext4_extent_header *eh,
459 int depth, ext4_fsblk_t pblk, ext4_lblk_t lblk)
460{
461 const char *error_msg;
462 int max = 0, err = -EFSCORRUPTED;
463
464 if (unlikely(eh->eh_magic != EXT4_EXT_MAGIC)) {
465 error_msg = "invalid magic";
466 goto corrupted;
467 }
468 if (unlikely(le16_to_cpu(eh->eh_depth) != depth)) {
469 error_msg = "unexpected eh_depth";
470 goto corrupted;
471 }
472 if (unlikely(eh->eh_max == 0)) {
473 error_msg = "invalid eh_max";
474 goto corrupted;
475 }
476 max = ext4_ext_max_entries(inode, depth);
477 if (unlikely(le16_to_cpu(eh->eh_max) > max)) {
478 error_msg = "too large eh_max";
479 goto corrupted;
480 }
481 if (unlikely(le16_to_cpu(eh->eh_entries) > le16_to_cpu(eh->eh_max))) {
482 error_msg = "invalid eh_entries";
483 goto corrupted;
484 }
485 if (unlikely((eh->eh_entries == 0) && (depth > 0))) {
486 error_msg = "eh_entries is 0 but eh_depth is > 0";
487 goto corrupted;
488 }
489 if (!ext4_valid_extent_entries(inode, eh, lblk, pblk: &pblk, depth)) {
490 error_msg = "invalid extent entries";
491 goto corrupted;
492 }
493 if (unlikely(depth > 32)) {
494 error_msg = "too large eh_depth";
495 goto corrupted;
496 }
497 /* Verify checksum on non-root extent tree nodes */
498 if (ext_depth(inode) != depth &&
499 !ext4_extent_block_csum_verify(inode, eh)) {
500 error_msg = "extent tree corrupted";
501 err = -EFSBADCRC;
502 goto corrupted;
503 }
504 return 0;
505
506corrupted:
507 ext4_error_inode_err(inode, function, line, 0, -err,
508 "pblk %llu bad header/extent: %s - magic %x, "
509 "entries %u, max %u(%u), depth %u(%u)",
510 (unsigned long long) pblk, error_msg,
511 le16_to_cpu(eh->eh_magic),
512 le16_to_cpu(eh->eh_entries),
513 le16_to_cpu(eh->eh_max),
514 max, le16_to_cpu(eh->eh_depth), depth);
515 return err;
516}
517
518#define ext4_ext_check(inode, eh, depth, pblk) \
519 __ext4_ext_check(__func__, __LINE__, (inode), (eh), (depth), (pblk), 0)
520
521int ext4_ext_check_inode(struct inode *inode)
522{
523 return ext4_ext_check(inode, ext_inode_hdr(inode), ext_depth(inode), 0);
524}
525
526static void ext4_cache_extents(struct inode *inode,
527 struct ext4_extent_header *eh)
528{
529 struct ext4_extent *ex = EXT_FIRST_EXTENT(eh);
530 ext4_lblk_t prev = 0;
531 int i;
532
533 for (i = le16_to_cpu(eh->eh_entries); i > 0; i--, ex++) {
534 unsigned int status = EXTENT_STATUS_WRITTEN;
535 ext4_lblk_t lblk = le32_to_cpu(ex->ee_block);
536 int len = ext4_ext_get_actual_len(ext: ex);
537
538 if (prev && (prev != lblk))
539 ext4_es_cache_extent(inode, lblk: prev, len: lblk - prev, pblk: ~0,
540 EXTENT_STATUS_HOLE);
541
542 if (ext4_ext_is_unwritten(ext: ex))
543 status = EXTENT_STATUS_UNWRITTEN;
544 ext4_es_cache_extent(inode, lblk, len,
545 pblk: ext4_ext_pblock(ex), status);
546 prev = lblk + len;
547 }
548}
549
550static struct buffer_head *
551__read_extent_tree_block(const char *function, unsigned int line,
552 struct inode *inode, struct ext4_extent_idx *idx,
553 int depth, int flags)
554{
555 struct buffer_head *bh;
556 int err;
557 gfp_t gfp_flags = __GFP_MOVABLE | GFP_NOFS;
558 ext4_fsblk_t pblk;
559
560 if (flags & EXT4_EX_NOFAIL)
561 gfp_flags |= __GFP_NOFAIL;
562
563 pblk = ext4_idx_pblock(ix: idx);
564 bh = sb_getblk_gfp(sb: inode->i_sb, block: pblk, gfp: gfp_flags);
565 if (unlikely(!bh))
566 return ERR_PTR(error: -ENOMEM);
567
568 if (!bh_uptodate_or_lock(bh)) {
569 trace_ext4_ext_load_extent(inode, lblk: pblk, _RET_IP_);
570 err = ext4_read_bh(bh, op_flags: 0, NULL, simu_fail: false);
571 if (err < 0)
572 goto errout;
573 }
574 if (buffer_verified(bh) && !(flags & EXT4_EX_FORCE_CACHE))
575 return bh;
576 err = __ext4_ext_check(function, line, inode, eh: ext_block_hdr(bh),
577 depth, pblk, le32_to_cpu(idx->ei_block));
578 if (err)
579 goto errout;
580 set_buffer_verified(bh);
581 /*
582 * If this is a leaf block, cache all of its entries
583 */
584 if (!(flags & EXT4_EX_NOCACHE) && depth == 0) {
585 struct ext4_extent_header *eh = ext_block_hdr(bh);
586 ext4_cache_extents(inode, eh);
587 }
588 return bh;
589errout:
590 put_bh(bh);
591 return ERR_PTR(error: err);
592
593}
594
595#define read_extent_tree_block(inode, idx, depth, flags) \
596 __read_extent_tree_block(__func__, __LINE__, (inode), (idx), \
597 (depth), (flags))
598
599/*
600 * This function is called to cache a file's extent information in the
601 * extent status tree
602 */
603int ext4_ext_precache(struct inode *inode)
604{
605 struct ext4_inode_info *ei = EXT4_I(inode);
606 struct ext4_ext_path *path = NULL;
607 struct buffer_head *bh;
608 int i = 0, depth, ret = 0;
609
610 if (!ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))
611 return 0; /* not an extent-mapped inode */
612
613 ext4_check_map_extents_env(inode);
614
615 down_read(sem: &ei->i_data_sem);
616 depth = ext_depth(inode);
617
618 /* Don't cache anything if there are no external extent blocks */
619 if (!depth) {
620 up_read(sem: &ei->i_data_sem);
621 return ret;
622 }
623
624 path = kcalloc(depth + 1, sizeof(struct ext4_ext_path),
625 GFP_NOFS);
626 if (path == NULL) {
627 up_read(sem: &ei->i_data_sem);
628 return -ENOMEM;
629 }
630
631 path[0].p_hdr = ext_inode_hdr(inode);
632 ret = ext4_ext_check(inode, path[0].p_hdr, depth, 0);
633 if (ret)
634 goto out;
635 path[0].p_idx = EXT_FIRST_INDEX(path[0].p_hdr);
636 while (i >= 0) {
637 /*
638 * If this is a leaf block or we've reached the end of
639 * the index block, go up
640 */
641 if ((i == depth) ||
642 path[i].p_idx > EXT_LAST_INDEX(path[i].p_hdr)) {
643 ext4_ext_path_brelse(path: path + i);
644 i--;
645 continue;
646 }
647 bh = read_extent_tree_block(inode, path[i].p_idx++,
648 depth - i - 1,
649 EXT4_EX_FORCE_CACHE);
650 if (IS_ERR(ptr: bh)) {
651 ret = PTR_ERR(ptr: bh);
652 break;
653 }
654 i++;
655 path[i].p_bh = bh;
656 path[i].p_hdr = ext_block_hdr(bh);
657 path[i].p_idx = EXT_FIRST_INDEX(path[i].p_hdr);
658 }
659 ext4_set_inode_state(inode, bit: EXT4_STATE_EXT_PRECACHED);
660out:
661 up_read(sem: &ei->i_data_sem);
662 ext4_free_ext_path(path);
663 return ret;
664}
665
666#ifdef EXT_DEBUG
667static void ext4_ext_show_path(struct inode *inode, struct ext4_ext_path *path)
668{
669 int k, l = path->p_depth;
670
671 ext_debug(inode, "path:");
672 for (k = 0; k <= l; k++, path++) {
673 if (path->p_idx) {
674 ext_debug(inode, " %d->%llu",
675 le32_to_cpu(path->p_idx->ei_block),
676 ext4_idx_pblock(path->p_idx));
677 } else if (path->p_ext) {
678 ext_debug(inode, " %d:[%d]%d:%llu ",
679 le32_to_cpu(path->p_ext->ee_block),
680 ext4_ext_is_unwritten(path->p_ext),
681 ext4_ext_get_actual_len(path->p_ext),
682 ext4_ext_pblock(path->p_ext));
683 } else
684 ext_debug(inode, " []");
685 }
686 ext_debug(inode, "\n");
687}
688
689static void ext4_ext_show_leaf(struct inode *inode, struct ext4_ext_path *path)
690{
691 int depth = ext_depth(inode);
692 struct ext4_extent_header *eh;
693 struct ext4_extent *ex;
694 int i;
695
696 if (IS_ERR_OR_NULL(path))
697 return;
698
699 eh = path[depth].p_hdr;
700 ex = EXT_FIRST_EXTENT(eh);
701
702 ext_debug(inode, "Displaying leaf extents\n");
703
704 for (i = 0; i < le16_to_cpu(eh->eh_entries); i++, ex++) {
705 ext_debug(inode, "%d:[%d]%d:%llu ", le32_to_cpu(ex->ee_block),
706 ext4_ext_is_unwritten(ex),
707 ext4_ext_get_actual_len(ex), ext4_ext_pblock(ex));
708 }
709 ext_debug(inode, "\n");
710}
711
712static void ext4_ext_show_move(struct inode *inode, struct ext4_ext_path *path,
713 ext4_fsblk_t newblock, int level)
714{
715 int depth = ext_depth(inode);
716 struct ext4_extent *ex;
717
718 if (depth != level) {
719 struct ext4_extent_idx *idx;
720 idx = path[level].p_idx;
721 while (idx <= EXT_MAX_INDEX(path[level].p_hdr)) {
722 ext_debug(inode, "%d: move %d:%llu in new index %llu\n",
723 level, le32_to_cpu(idx->ei_block),
724 ext4_idx_pblock(idx), newblock);
725 idx++;
726 }
727
728 return;
729 }
730
731 ex = path[depth].p_ext;
732 while (ex <= EXT_MAX_EXTENT(path[depth].p_hdr)) {
733 ext_debug(inode, "move %d:%llu:[%d]%d in new leaf %llu\n",
734 le32_to_cpu(ex->ee_block),
735 ext4_ext_pblock(ex),
736 ext4_ext_is_unwritten(ex),
737 ext4_ext_get_actual_len(ex),
738 newblock);
739 ex++;
740 }
741}
742
743#else
744#define ext4_ext_show_path(inode, path)
745#define ext4_ext_show_leaf(inode, path)
746#define ext4_ext_show_move(inode, path, newblock, level)
747#endif
748
749/*
750 * ext4_ext_binsearch_idx:
751 * binary search for the closest index of the given block
752 * the header must be checked before calling this
753 */
754static void
755ext4_ext_binsearch_idx(struct inode *inode,
756 struct ext4_ext_path *path, ext4_lblk_t block)
757{
758 struct ext4_extent_header *eh = path->p_hdr;
759 struct ext4_extent_idx *r, *l, *m;
760
761
762 ext_debug(inode, "binsearch for %u(idx): ", block);
763
764 l = EXT_FIRST_INDEX(eh) + 1;
765 r = EXT_LAST_INDEX(eh);
766 while (l <= r) {
767 m = l + (r - l) / 2;
768 ext_debug(inode, "%p(%u):%p(%u):%p(%u) ", l,
769 le32_to_cpu(l->ei_block), m, le32_to_cpu(m->ei_block),
770 r, le32_to_cpu(r->ei_block));
771
772 if (block < le32_to_cpu(m->ei_block))
773 r = m - 1;
774 else
775 l = m + 1;
776 }
777
778 path->p_idx = l - 1;
779 ext_debug(inode, " -> %u->%lld ", le32_to_cpu(path->p_idx->ei_block),
780 ext4_idx_pblock(path->p_idx));
781
782#ifdef CHECK_BINSEARCH
783 {
784 struct ext4_extent_idx *chix, *ix;
785 int k;
786
787 chix = ix = EXT_FIRST_INDEX(eh);
788 for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ix++) {
789 if (k != 0 && le32_to_cpu(ix->ei_block) <=
790 le32_to_cpu(ix[-1].ei_block)) {
791 printk(KERN_DEBUG "k=%d, ix=0x%p, "
792 "first=0x%p\n", k,
793 ix, EXT_FIRST_INDEX(eh));
794 printk(KERN_DEBUG "%u <= %u\n",
795 le32_to_cpu(ix->ei_block),
796 le32_to_cpu(ix[-1].ei_block));
797 }
798 BUG_ON(k && le32_to_cpu(ix->ei_block)
799 <= le32_to_cpu(ix[-1].ei_block));
800 if (block < le32_to_cpu(ix->ei_block))
801 break;
802 chix = ix;
803 }
804 BUG_ON(chix != path->p_idx);
805 }
806#endif
807
808}
809
810/*
811 * ext4_ext_binsearch:
812 * binary search for closest extent of the given block
813 * the header must be checked before calling this
814 */
815static void
816ext4_ext_binsearch(struct inode *inode,
817 struct ext4_ext_path *path, ext4_lblk_t block)
818{
819 struct ext4_extent_header *eh = path->p_hdr;
820 struct ext4_extent *r, *l, *m;
821
822 if (eh->eh_entries == 0) {
823 /*
824 * this leaf is empty:
825 * we get such a leaf in split/add case
826 */
827 return;
828 }
829
830 ext_debug(inode, "binsearch for %u: ", block);
831
832 l = EXT_FIRST_EXTENT(eh) + 1;
833 r = EXT_LAST_EXTENT(eh);
834
835 while (l <= r) {
836 m = l + (r - l) / 2;
837 ext_debug(inode, "%p(%u):%p(%u):%p(%u) ", l,
838 le32_to_cpu(l->ee_block), m, le32_to_cpu(m->ee_block),
839 r, le32_to_cpu(r->ee_block));
840
841 if (block < le32_to_cpu(m->ee_block))
842 r = m - 1;
843 else
844 l = m + 1;
845 }
846
847 path->p_ext = l - 1;
848 ext_debug(inode, " -> %d:%llu:[%d]%d ",
849 le32_to_cpu(path->p_ext->ee_block),
850 ext4_ext_pblock(path->p_ext),
851 ext4_ext_is_unwritten(path->p_ext),
852 ext4_ext_get_actual_len(path->p_ext));
853
854#ifdef CHECK_BINSEARCH
855 {
856 struct ext4_extent *chex, *ex;
857 int k;
858
859 chex = ex = EXT_FIRST_EXTENT(eh);
860 for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ex++) {
861 BUG_ON(k && le32_to_cpu(ex->ee_block)
862 <= le32_to_cpu(ex[-1].ee_block));
863 if (block < le32_to_cpu(ex->ee_block))
864 break;
865 chex = ex;
866 }
867 BUG_ON(chex != path->p_ext);
868 }
869#endif
870
871}
872
873void ext4_ext_tree_init(handle_t *handle, struct inode *inode)
874{
875 struct ext4_extent_header *eh;
876
877 eh = ext_inode_hdr(inode);
878 eh->eh_depth = 0;
879 eh->eh_entries = 0;
880 eh->eh_magic = EXT4_EXT_MAGIC;
881 eh->eh_max = cpu_to_le16(ext4_ext_space_root(inode, 0));
882 eh->eh_generation = 0;
883 ext4_mark_inode_dirty(handle, inode);
884}
885
886struct ext4_ext_path *
887ext4_find_extent(struct inode *inode, ext4_lblk_t block,
888 struct ext4_ext_path *path, int flags)
889{
890 struct ext4_extent_header *eh;
891 struct buffer_head *bh;
892 short int depth, i, ppos = 0;
893 int ret;
894 gfp_t gfp_flags = GFP_NOFS;
895
896 if (flags & EXT4_EX_NOFAIL)
897 gfp_flags |= __GFP_NOFAIL;
898
899 eh = ext_inode_hdr(inode);
900 depth = ext_depth(inode);
901 if (depth < 0 || depth > EXT4_MAX_EXTENT_DEPTH) {
902 EXT4_ERROR_INODE(inode, "inode has invalid extent depth: %d",
903 depth);
904 ret = -EFSCORRUPTED;
905 goto err;
906 }
907
908 if (path) {
909 ext4_ext_drop_refs(path);
910 if (depth > path[0].p_maxdepth) {
911 kfree(objp: path);
912 path = NULL;
913 }
914 }
915 if (!path) {
916 /* account possible depth increase */
917 path = kcalloc(depth + 2, sizeof(struct ext4_ext_path),
918 gfp_flags);
919 if (unlikely(!path))
920 return ERR_PTR(error: -ENOMEM);
921 path[0].p_maxdepth = depth + 1;
922 }
923 path[0].p_hdr = eh;
924 path[0].p_bh = NULL;
925
926 i = depth;
927 if (!(flags & EXT4_EX_NOCACHE) && depth == 0)
928 ext4_cache_extents(inode, eh);
929 /* walk through the tree */
930 while (i) {
931 ext_debug(inode, "depth %d: num %d, max %d\n",
932 ppos, le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max));
933
934 ext4_ext_binsearch_idx(inode, path: path + ppos, block);
935 path[ppos].p_block = ext4_idx_pblock(ix: path[ppos].p_idx);
936 path[ppos].p_depth = i;
937 path[ppos].p_ext = NULL;
938
939 bh = read_extent_tree_block(inode, path[ppos].p_idx, --i, flags);
940 if (IS_ERR(ptr: bh)) {
941 ret = PTR_ERR(ptr: bh);
942 goto err;
943 }
944
945 eh = ext_block_hdr(bh);
946 ppos++;
947 path[ppos].p_bh = bh;
948 path[ppos].p_hdr = eh;
949 }
950
951 path[ppos].p_depth = i;
952 path[ppos].p_ext = NULL;
953 path[ppos].p_idx = NULL;
954
955 /* find extent */
956 ext4_ext_binsearch(inode, path: path + ppos, block);
957 /* if not an empty leaf */
958 if (path[ppos].p_ext)
959 path[ppos].p_block = ext4_ext_pblock(ex: path[ppos].p_ext);
960
961 ext4_ext_show_path(inode, path);
962
963 return path;
964
965err:
966 ext4_free_ext_path(path);
967 return ERR_PTR(error: ret);
968}
969
970/*
971 * ext4_ext_insert_index:
972 * insert new index [@logical;@ptr] into the block at @curp;
973 * check where to insert: before @curp or after @curp
974 */
975static int ext4_ext_insert_index(handle_t *handle, struct inode *inode,
976 struct ext4_ext_path *curp,
977 int logical, ext4_fsblk_t ptr)
978{
979 struct ext4_extent_idx *ix;
980 int len, err;
981
982 err = ext4_ext_get_access(handle, inode, path: curp);
983 if (err)
984 return err;
985
986 if (unlikely(logical == le32_to_cpu(curp->p_idx->ei_block))) {
987 EXT4_ERROR_INODE(inode,
988 "logical %d == ei_block %d!",
989 logical, le32_to_cpu(curp->p_idx->ei_block));
990 return -EFSCORRUPTED;
991 }
992
993 if (unlikely(le16_to_cpu(curp->p_hdr->eh_entries)
994 >= le16_to_cpu(curp->p_hdr->eh_max))) {
995 EXT4_ERROR_INODE(inode,
996 "eh_entries %d >= eh_max %d!",
997 le16_to_cpu(curp->p_hdr->eh_entries),
998 le16_to_cpu(curp->p_hdr->eh_max));
999 return -EFSCORRUPTED;
1000 }
1001
1002 if (logical > le32_to_cpu(curp->p_idx->ei_block)) {
1003 /* insert after */
1004 ext_debug(inode, "insert new index %d after: %llu\n",
1005 logical, ptr);
1006 ix = curp->p_idx + 1;
1007 } else {
1008 /* insert before */
1009 ext_debug(inode, "insert new index %d before: %llu\n",
1010 logical, ptr);
1011 ix = curp->p_idx;
1012 }
1013
1014 if (unlikely(ix > EXT_MAX_INDEX(curp->p_hdr))) {
1015 EXT4_ERROR_INODE(inode, "ix > EXT_MAX_INDEX!");
1016 return -EFSCORRUPTED;
1017 }
1018
1019 len = EXT_LAST_INDEX(curp->p_hdr) - ix + 1;
1020 BUG_ON(len < 0);
1021 if (len > 0) {
1022 ext_debug(inode, "insert new index %d: "
1023 "move %d indices from 0x%p to 0x%p\n",
1024 logical, len, ix, ix + 1);
1025 memmove(dest: ix + 1, src: ix, count: len * sizeof(struct ext4_extent_idx));
1026 }
1027
1028 ix->ei_block = cpu_to_le32(logical);
1029 ext4_idx_store_pblock(ix, pb: ptr);
1030 le16_add_cpu(var: &curp->p_hdr->eh_entries, val: 1);
1031
1032 if (unlikely(ix > EXT_LAST_INDEX(curp->p_hdr))) {
1033 EXT4_ERROR_INODE(inode, "ix > EXT_LAST_INDEX!");
1034 return -EFSCORRUPTED;
1035 }
1036
1037 err = ext4_ext_dirty(handle, inode, curp);
1038 ext4_std_error(inode->i_sb, err);
1039
1040 return err;
1041}
1042
1043/*
1044 * ext4_ext_split:
1045 * inserts new subtree into the path, using free index entry
1046 * at depth @at:
1047 * - allocates all needed blocks (new leaf and all intermediate index blocks)
1048 * - makes decision where to split
1049 * - moves remaining extents and index entries (right to the split point)
1050 * into the newly allocated blocks
1051 * - initializes subtree
1052 */
1053static int ext4_ext_split(handle_t *handle, struct inode *inode,
1054 unsigned int flags,
1055 struct ext4_ext_path *path,
1056 struct ext4_extent *newext, int at)
1057{
1058 struct buffer_head *bh = NULL;
1059 int depth = ext_depth(inode);
1060 struct ext4_extent_header *neh;
1061 struct ext4_extent_idx *fidx;
1062 int i = at, k, m, a;
1063 ext4_fsblk_t newblock, oldblock;
1064 __le32 border;
1065 ext4_fsblk_t *ablocks = NULL; /* array of allocated blocks */
1066 gfp_t gfp_flags = GFP_NOFS;
1067 int err = 0;
1068 size_t ext_size = 0;
1069
1070 if (flags & EXT4_EX_NOFAIL)
1071 gfp_flags |= __GFP_NOFAIL;
1072
1073 /* make decision: where to split? */
1074 /* FIXME: now decision is simplest: at current extent */
1075
1076 /* if current leaf will be split, then we should use
1077 * border from split point */
1078 if (unlikely(path[depth].p_ext > EXT_MAX_EXTENT(path[depth].p_hdr))) {
1079 EXT4_ERROR_INODE(inode, "p_ext > EXT_MAX_EXTENT!");
1080 return -EFSCORRUPTED;
1081 }
1082 if (path[depth].p_ext != EXT_MAX_EXTENT(path[depth].p_hdr)) {
1083 border = path[depth].p_ext[1].ee_block;
1084 ext_debug(inode, "leaf will be split."
1085 " next leaf starts at %d\n",
1086 le32_to_cpu(border));
1087 } else {
1088 border = newext->ee_block;
1089 ext_debug(inode, "leaf will be added."
1090 " next leaf starts at %d\n",
1091 le32_to_cpu(border));
1092 }
1093
1094 /*
1095 * If error occurs, then we break processing
1096 * and mark filesystem read-only. index won't
1097 * be inserted and tree will be in consistent
1098 * state. Next mount will repair buffers too.
1099 */
1100
1101 /*
1102 * Get array to track all allocated blocks.
1103 * We need this to handle errors and free blocks
1104 * upon them.
1105 */
1106 ablocks = kcalloc(depth, sizeof(ext4_fsblk_t), gfp_flags);
1107 if (!ablocks)
1108 return -ENOMEM;
1109
1110 /* allocate all needed blocks */
1111 ext_debug(inode, "allocate %d blocks for indexes/leaf\n", depth - at);
1112 for (a = 0; a < depth - at; a++) {
1113 newblock = ext4_ext_new_meta_block(handle, inode, path,
1114 ex: newext, err: &err, flags);
1115 if (newblock == 0)
1116 goto cleanup;
1117 ablocks[a] = newblock;
1118 }
1119
1120 /* initialize new leaf */
1121 newblock = ablocks[--a];
1122 if (unlikely(newblock == 0)) {
1123 EXT4_ERROR_INODE(inode, "newblock == 0!");
1124 err = -EFSCORRUPTED;
1125 goto cleanup;
1126 }
1127 bh = sb_getblk_gfp(sb: inode->i_sb, block: newblock, __GFP_MOVABLE | GFP_NOFS);
1128 if (unlikely(!bh)) {
1129 err = -ENOMEM;
1130 goto cleanup;
1131 }
1132 lock_buffer(bh);
1133
1134 err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
1135 EXT4_JTR_NONE);
1136 if (err)
1137 goto cleanup;
1138
1139 neh = ext_block_hdr(bh);
1140 neh->eh_entries = 0;
1141 neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0));
1142 neh->eh_magic = EXT4_EXT_MAGIC;
1143 neh->eh_depth = 0;
1144 neh->eh_generation = 0;
1145
1146 /* move remainder of path[depth] to the new leaf */
1147 if (unlikely(path[depth].p_hdr->eh_entries !=
1148 path[depth].p_hdr->eh_max)) {
1149 EXT4_ERROR_INODE(inode, "eh_entries %d != eh_max %d!",
1150 path[depth].p_hdr->eh_entries,
1151 path[depth].p_hdr->eh_max);
1152 err = -EFSCORRUPTED;
1153 goto cleanup;
1154 }
1155 /* start copy from next extent */
1156 m = EXT_MAX_EXTENT(path[depth].p_hdr) - path[depth].p_ext++;
1157 ext4_ext_show_move(inode, path, newblock, depth);
1158 if (m) {
1159 struct ext4_extent *ex;
1160 ex = EXT_FIRST_EXTENT(neh);
1161 memmove(dest: ex, src: path[depth].p_ext, count: sizeof(struct ext4_extent) * m);
1162 le16_add_cpu(var: &neh->eh_entries, val: m);
1163 }
1164
1165 /* zero out unused area in the extent block */
1166 ext_size = sizeof(struct ext4_extent_header) +
1167 sizeof(struct ext4_extent) * le16_to_cpu(neh->eh_entries);
1168 memset(s: bh->b_data + ext_size, c: 0, n: inode->i_sb->s_blocksize - ext_size);
1169 ext4_extent_block_csum_set(inode, eh: neh);
1170 set_buffer_uptodate(bh);
1171 unlock_buffer(bh);
1172
1173 err = ext4_handle_dirty_metadata(handle, inode, bh);
1174 if (err)
1175 goto cleanup;
1176 brelse(bh);
1177 bh = NULL;
1178
1179 /* correct old leaf */
1180 if (m) {
1181 err = ext4_ext_get_access(handle, inode, path: path + depth);
1182 if (err)
1183 goto cleanup;
1184 le16_add_cpu(var: &path[depth].p_hdr->eh_entries, val: -m);
1185 err = ext4_ext_dirty(handle, inode, path + depth);
1186 if (err)
1187 goto cleanup;
1188
1189 }
1190
1191 /* create intermediate indexes */
1192 k = depth - at - 1;
1193 if (unlikely(k < 0)) {
1194 EXT4_ERROR_INODE(inode, "k %d < 0!", k);
1195 err = -EFSCORRUPTED;
1196 goto cleanup;
1197 }
1198 if (k)
1199 ext_debug(inode, "create %d intermediate indices\n", k);
1200 /* insert new index into current index block */
1201 /* current depth stored in i var */
1202 i = depth - 1;
1203 while (k--) {
1204 oldblock = newblock;
1205 newblock = ablocks[--a];
1206 bh = sb_getblk(sb: inode->i_sb, block: newblock);
1207 if (unlikely(!bh)) {
1208 err = -ENOMEM;
1209 goto cleanup;
1210 }
1211 lock_buffer(bh);
1212
1213 err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
1214 EXT4_JTR_NONE);
1215 if (err)
1216 goto cleanup;
1217
1218 neh = ext_block_hdr(bh);
1219 neh->eh_entries = cpu_to_le16(1);
1220 neh->eh_magic = EXT4_EXT_MAGIC;
1221 neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0));
1222 neh->eh_depth = cpu_to_le16(depth - i);
1223 neh->eh_generation = 0;
1224 fidx = EXT_FIRST_INDEX(neh);
1225 fidx->ei_block = border;
1226 ext4_idx_store_pblock(ix: fidx, pb: oldblock);
1227
1228 ext_debug(inode, "int.index at %d (block %llu): %u -> %llu\n",
1229 i, newblock, le32_to_cpu(border), oldblock);
1230
1231 /* move remainder of path[i] to the new index block */
1232 if (unlikely(EXT_MAX_INDEX(path[i].p_hdr) !=
1233 EXT_LAST_INDEX(path[i].p_hdr))) {
1234 EXT4_ERROR_INODE(inode,
1235 "EXT_MAX_INDEX != EXT_LAST_INDEX ee_block %d!",
1236 le32_to_cpu(path[i].p_ext->ee_block));
1237 err = -EFSCORRUPTED;
1238 goto cleanup;
1239 }
1240 /* start copy indexes */
1241 m = EXT_MAX_INDEX(path[i].p_hdr) - path[i].p_idx++;
1242 ext_debug(inode, "cur 0x%p, last 0x%p\n", path[i].p_idx,
1243 EXT_MAX_INDEX(path[i].p_hdr));
1244 ext4_ext_show_move(inode, path, newblock, i);
1245 if (m) {
1246 memmove(dest: ++fidx, src: path[i].p_idx,
1247 count: sizeof(struct ext4_extent_idx) * m);
1248 le16_add_cpu(var: &neh->eh_entries, val: m);
1249 }
1250 /* zero out unused area in the extent block */
1251 ext_size = sizeof(struct ext4_extent_header) +
1252 (sizeof(struct ext4_extent) * le16_to_cpu(neh->eh_entries));
1253 memset(s: bh->b_data + ext_size, c: 0,
1254 n: inode->i_sb->s_blocksize - ext_size);
1255 ext4_extent_block_csum_set(inode, eh: neh);
1256 set_buffer_uptodate(bh);
1257 unlock_buffer(bh);
1258
1259 err = ext4_handle_dirty_metadata(handle, inode, bh);
1260 if (err)
1261 goto cleanup;
1262 brelse(bh);
1263 bh = NULL;
1264
1265 /* correct old index */
1266 if (m) {
1267 err = ext4_ext_get_access(handle, inode, path: path + i);
1268 if (err)
1269 goto cleanup;
1270 le16_add_cpu(var: &path[i].p_hdr->eh_entries, val: -m);
1271 err = ext4_ext_dirty(handle, inode, path + i);
1272 if (err)
1273 goto cleanup;
1274 }
1275
1276 i--;
1277 }
1278
1279 /* insert new index */
1280 err = ext4_ext_insert_index(handle, inode, curp: path + at,
1281 le32_to_cpu(border), ptr: newblock);
1282
1283cleanup:
1284 if (bh) {
1285 if (buffer_locked(bh))
1286 unlock_buffer(bh);
1287 brelse(bh);
1288 }
1289
1290 if (err) {
1291 /* free all allocated blocks in error case */
1292 for (i = 0; i < depth; i++) {
1293 if (!ablocks[i])
1294 continue;
1295 ext4_free_blocks(handle, inode, NULL, block: ablocks[i], count: 1,
1296 EXT4_FREE_BLOCKS_METADATA);
1297 }
1298 }
1299 kfree(objp: ablocks);
1300
1301 return err;
1302}
1303
1304/*
1305 * ext4_ext_grow_indepth:
1306 * implements tree growing procedure:
1307 * - allocates new block
1308 * - moves top-level data (index block or leaf) into the new block
1309 * - initializes new top-level, creating index that points to the
1310 * just created block
1311 */
1312static int ext4_ext_grow_indepth(handle_t *handle, struct inode *inode,
1313 unsigned int flags)
1314{
1315 struct ext4_extent_header *neh;
1316 struct buffer_head *bh;
1317 ext4_fsblk_t newblock, goal = 0;
1318 struct ext4_super_block *es = EXT4_SB(sb: inode->i_sb)->s_es;
1319 int err = 0;
1320 size_t ext_size = 0;
1321
1322 /* Try to prepend new index to old one */
1323 if (ext_depth(inode))
1324 goal = ext4_idx_pblock(EXT_FIRST_INDEX(ext_inode_hdr(inode)));
1325 if (goal > le32_to_cpu(es->s_first_data_block)) {
1326 flags |= EXT4_MB_HINT_TRY_GOAL;
1327 goal--;
1328 } else
1329 goal = ext4_inode_to_goal_block(inode);
1330 newblock = ext4_new_meta_blocks(handle, inode, goal, flags,
1331 NULL, errp: &err);
1332 if (newblock == 0)
1333 return err;
1334
1335 bh = sb_getblk_gfp(sb: inode->i_sb, block: newblock, __GFP_MOVABLE | GFP_NOFS);
1336 if (unlikely(!bh))
1337 return -ENOMEM;
1338 lock_buffer(bh);
1339
1340 err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
1341 EXT4_JTR_NONE);
1342 if (err) {
1343 unlock_buffer(bh);
1344 goto out;
1345 }
1346
1347 ext_size = sizeof(EXT4_I(inode)->i_data);
1348 /* move top-level index/leaf into new block */
1349 memmove(dest: bh->b_data, EXT4_I(inode)->i_data, count: ext_size);
1350 /* zero out unused area in the extent block */
1351 memset(s: bh->b_data + ext_size, c: 0, n: inode->i_sb->s_blocksize - ext_size);
1352
1353 /* set size of new block */
1354 neh = ext_block_hdr(bh);
1355 /* old root could have indexes or leaves
1356 * so calculate e_max right way */
1357 if (ext_depth(inode))
1358 neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0));
1359 else
1360 neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0));
1361 neh->eh_magic = EXT4_EXT_MAGIC;
1362 ext4_extent_block_csum_set(inode, eh: neh);
1363 set_buffer_uptodate(bh);
1364 set_buffer_verified(bh);
1365 unlock_buffer(bh);
1366
1367 err = ext4_handle_dirty_metadata(handle, inode, bh);
1368 if (err)
1369 goto out;
1370
1371 /* Update top-level index: num,max,pointer */
1372 neh = ext_inode_hdr(inode);
1373 neh->eh_entries = cpu_to_le16(1);
1374 ext4_idx_store_pblock(EXT_FIRST_INDEX(neh), pb: newblock);
1375 if (neh->eh_depth == 0) {
1376 /* Root extent block becomes index block */
1377 neh->eh_max = cpu_to_le16(ext4_ext_space_root_idx(inode, 0));
1378 EXT_FIRST_INDEX(neh)->ei_block =
1379 EXT_FIRST_EXTENT(neh)->ee_block;
1380 }
1381 ext_debug(inode, "new root: num %d(%d), lblock %d, ptr %llu\n",
1382 le16_to_cpu(neh->eh_entries), le16_to_cpu(neh->eh_max),
1383 le32_to_cpu(EXT_FIRST_INDEX(neh)->ei_block),
1384 ext4_idx_pblock(EXT_FIRST_INDEX(neh)));
1385
1386 le16_add_cpu(var: &neh->eh_depth, val: 1);
1387 err = ext4_mark_inode_dirty(handle, inode);
1388out:
1389 brelse(bh);
1390
1391 return err;
1392}
1393
1394/*
1395 * ext4_ext_create_new_leaf:
1396 * finds empty index and adds new leaf.
1397 * if no free index is found, then it requests in-depth growing.
1398 */
1399static struct ext4_ext_path *
1400ext4_ext_create_new_leaf(handle_t *handle, struct inode *inode,
1401 unsigned int mb_flags, unsigned int gb_flags,
1402 struct ext4_ext_path *path,
1403 struct ext4_extent *newext)
1404{
1405 struct ext4_ext_path *curp;
1406 int depth, i, err = 0;
1407 ext4_lblk_t ee_block = le32_to_cpu(newext->ee_block);
1408
1409repeat:
1410 i = depth = ext_depth(inode);
1411
1412 /* walk up to the tree and look for free index entry */
1413 curp = path + depth;
1414 while (i > 0 && !EXT_HAS_FREE_INDEX(curp)) {
1415 i--;
1416 curp--;
1417 }
1418
1419 /* we use already allocated block for index block,
1420 * so subsequent data blocks should be contiguous */
1421 if (EXT_HAS_FREE_INDEX(curp)) {
1422 /* if we found index with free entry, then use that
1423 * entry: create all needed subtree and add new leaf */
1424 err = ext4_ext_split(handle, inode, flags: mb_flags, path, newext, at: i);
1425 if (err)
1426 goto errout;
1427
1428 /* refill path */
1429 path = ext4_find_extent(inode, block: ee_block, path, flags: gb_flags);
1430 return path;
1431 }
1432
1433 /* tree is full, time to grow in depth */
1434 err = ext4_ext_grow_indepth(handle, inode, flags: mb_flags);
1435 if (err)
1436 goto errout;
1437
1438 /* refill path */
1439 path = ext4_find_extent(inode, block: ee_block, path, flags: gb_flags);
1440 if (IS_ERR(ptr: path))
1441 return path;
1442
1443 /*
1444 * only first (depth 0 -> 1) produces free space;
1445 * in all other cases we have to split the grown tree
1446 */
1447 depth = ext_depth(inode);
1448 if (path[depth].p_hdr->eh_entries == path[depth].p_hdr->eh_max) {
1449 /* now we need to split */
1450 goto repeat;
1451 }
1452
1453 return path;
1454
1455errout:
1456 ext4_free_ext_path(path);
1457 return ERR_PTR(error: err);
1458}
1459
1460/*
1461 * search the closest allocated block to the left for *logical
1462 * and returns it at @logical + it's physical address at @phys
1463 * if *logical is the smallest allocated block, the function
1464 * returns 0 at @phys
1465 * return value contains 0 (success) or error code
1466 */
1467static int ext4_ext_search_left(struct inode *inode,
1468 struct ext4_ext_path *path,
1469 ext4_lblk_t *logical, ext4_fsblk_t *phys)
1470{
1471 struct ext4_extent_idx *ix;
1472 struct ext4_extent *ex;
1473 int depth, ee_len;
1474
1475 if (unlikely(path == NULL)) {
1476 EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical);
1477 return -EFSCORRUPTED;
1478 }
1479 depth = path->p_depth;
1480 *phys = 0;
1481
1482 if (depth == 0 && path->p_ext == NULL)
1483 return 0;
1484
1485 /* usually extent in the path covers blocks smaller
1486 * then *logical, but it can be that extent is the
1487 * first one in the file */
1488
1489 ex = path[depth].p_ext;
1490 ee_len = ext4_ext_get_actual_len(ext: ex);
1491 if (*logical < le32_to_cpu(ex->ee_block)) {
1492 if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) {
1493 EXT4_ERROR_INODE(inode,
1494 "EXT_FIRST_EXTENT != ex *logical %d ee_block %d!",
1495 *logical, le32_to_cpu(ex->ee_block));
1496 return -EFSCORRUPTED;
1497 }
1498 while (--depth >= 0) {
1499 ix = path[depth].p_idx;
1500 if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) {
1501 EXT4_ERROR_INODE(inode,
1502 "ix (%d) != EXT_FIRST_INDEX (%d) (depth %d)!",
1503 ix != NULL ? le32_to_cpu(ix->ei_block) : 0,
1504 le32_to_cpu(EXT_FIRST_INDEX(path[depth].p_hdr)->ei_block),
1505 depth);
1506 return -EFSCORRUPTED;
1507 }
1508 }
1509 return 0;
1510 }
1511
1512 if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) {
1513 EXT4_ERROR_INODE(inode,
1514 "logical %d < ee_block %d + ee_len %d!",
1515 *logical, le32_to_cpu(ex->ee_block), ee_len);
1516 return -EFSCORRUPTED;
1517 }
1518
1519 *logical = le32_to_cpu(ex->ee_block) + ee_len - 1;
1520 *phys = ext4_ext_pblock(ex) + ee_len - 1;
1521 return 0;
1522}
1523
1524/*
1525 * Search the closest allocated block to the right for *logical
1526 * and returns it at @logical + it's physical address at @phys.
1527 * If not exists, return 0 and @phys is set to 0. We will return
1528 * 1 which means we found an allocated block and ret_ex is valid.
1529 * Or return a (< 0) error code.
1530 */
1531static int ext4_ext_search_right(struct inode *inode,
1532 struct ext4_ext_path *path,
1533 ext4_lblk_t *logical, ext4_fsblk_t *phys,
1534 struct ext4_extent *ret_ex, int flags)
1535{
1536 struct buffer_head *bh = NULL;
1537 struct ext4_extent_header *eh;
1538 struct ext4_extent_idx *ix;
1539 struct ext4_extent *ex;
1540 int depth; /* Note, NOT eh_depth; depth from top of tree */
1541 int ee_len;
1542
1543 if (unlikely(path == NULL)) {
1544 EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical);
1545 return -EFSCORRUPTED;
1546 }
1547 depth = path->p_depth;
1548 *phys = 0;
1549
1550 if (depth == 0 && path->p_ext == NULL)
1551 return 0;
1552
1553 /* usually extent in the path covers blocks smaller
1554 * then *logical, but it can be that extent is the
1555 * first one in the file */
1556
1557 ex = path[depth].p_ext;
1558 ee_len = ext4_ext_get_actual_len(ext: ex);
1559 if (*logical < le32_to_cpu(ex->ee_block)) {
1560 if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) {
1561 EXT4_ERROR_INODE(inode,
1562 "first_extent(path[%d].p_hdr) != ex",
1563 depth);
1564 return -EFSCORRUPTED;
1565 }
1566 while (--depth >= 0) {
1567 ix = path[depth].p_idx;
1568 if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) {
1569 EXT4_ERROR_INODE(inode,
1570 "ix != EXT_FIRST_INDEX *logical %d!",
1571 *logical);
1572 return -EFSCORRUPTED;
1573 }
1574 }
1575 goto found_extent;
1576 }
1577
1578 if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) {
1579 EXT4_ERROR_INODE(inode,
1580 "logical %d < ee_block %d + ee_len %d!",
1581 *logical, le32_to_cpu(ex->ee_block), ee_len);
1582 return -EFSCORRUPTED;
1583 }
1584
1585 if (ex != EXT_LAST_EXTENT(path[depth].p_hdr)) {
1586 /* next allocated block in this leaf */
1587 ex++;
1588 goto found_extent;
1589 }
1590
1591 /* go up and search for index to the right */
1592 while (--depth >= 0) {
1593 ix = path[depth].p_idx;
1594 if (ix != EXT_LAST_INDEX(path[depth].p_hdr))
1595 goto got_index;
1596 }
1597
1598 /* we've gone up to the root and found no index to the right */
1599 return 0;
1600
1601got_index:
1602 /* we've found index to the right, let's
1603 * follow it and find the closest allocated
1604 * block to the right */
1605 ix++;
1606 while (++depth < path->p_depth) {
1607 /* subtract from p_depth to get proper eh_depth */
1608 bh = read_extent_tree_block(inode, ix, path->p_depth - depth,
1609 flags);
1610 if (IS_ERR(ptr: bh))
1611 return PTR_ERR(ptr: bh);
1612 eh = ext_block_hdr(bh);
1613 ix = EXT_FIRST_INDEX(eh);
1614 put_bh(bh);
1615 }
1616
1617 bh = read_extent_tree_block(inode, ix, path->p_depth - depth, flags);
1618 if (IS_ERR(ptr: bh))
1619 return PTR_ERR(ptr: bh);
1620 eh = ext_block_hdr(bh);
1621 ex = EXT_FIRST_EXTENT(eh);
1622found_extent:
1623 *logical = le32_to_cpu(ex->ee_block);
1624 *phys = ext4_ext_pblock(ex);
1625 if (ret_ex)
1626 *ret_ex = *ex;
1627 if (bh)
1628 put_bh(bh);
1629 return 1;
1630}
1631
1632/*
1633 * ext4_ext_next_allocated_block:
1634 * returns allocated block in subsequent extent or EXT_MAX_BLOCKS.
1635 * NOTE: it considers block number from index entry as
1636 * allocated block. Thus, index entries have to be consistent
1637 * with leaves.
1638 */
1639ext4_lblk_t
1640ext4_ext_next_allocated_block(struct ext4_ext_path *path)
1641{
1642 int depth;
1643
1644 BUG_ON(path == NULL);
1645 depth = path->p_depth;
1646
1647 if (depth == 0 && path->p_ext == NULL)
1648 return EXT_MAX_BLOCKS;
1649
1650 while (depth >= 0) {
1651 struct ext4_ext_path *p = &path[depth];
1652
1653 if (depth == path->p_depth) {
1654 /* leaf */
1655 if (p->p_ext && p->p_ext != EXT_LAST_EXTENT(p->p_hdr))
1656 return le32_to_cpu(p->p_ext[1].ee_block);
1657 } else {
1658 /* index */
1659 if (p->p_idx != EXT_LAST_INDEX(p->p_hdr))
1660 return le32_to_cpu(p->p_idx[1].ei_block);
1661 }
1662 depth--;
1663 }
1664
1665 return EXT_MAX_BLOCKS;
1666}
1667
1668/*
1669 * ext4_ext_next_leaf_block:
1670 * returns first allocated block from next leaf or EXT_MAX_BLOCKS
1671 */
1672static ext4_lblk_t ext4_ext_next_leaf_block(struct ext4_ext_path *path)
1673{
1674 int depth;
1675
1676 BUG_ON(path == NULL);
1677 depth = path->p_depth;
1678
1679 /* zero-tree has no leaf blocks at all */
1680 if (depth == 0)
1681 return EXT_MAX_BLOCKS;
1682
1683 /* go to index block */
1684 depth--;
1685
1686 while (depth >= 0) {
1687 if (path[depth].p_idx !=
1688 EXT_LAST_INDEX(path[depth].p_hdr))
1689 return (ext4_lblk_t)
1690 le32_to_cpu(path[depth].p_idx[1].ei_block);
1691 depth--;
1692 }
1693
1694 return EXT_MAX_BLOCKS;
1695}
1696
1697/*
1698 * ext4_ext_correct_indexes:
1699 * if leaf gets modified and modified extent is first in the leaf,
1700 * then we have to correct all indexes above.
1701 * TODO: do we need to correct tree in all cases?
1702 */
1703static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode,
1704 struct ext4_ext_path *path)
1705{
1706 struct ext4_extent_header *eh;
1707 int depth = ext_depth(inode);
1708 struct ext4_extent *ex;
1709 __le32 border;
1710 int k, err = 0;
1711
1712 eh = path[depth].p_hdr;
1713 ex = path[depth].p_ext;
1714
1715 if (unlikely(ex == NULL || eh == NULL)) {
1716 EXT4_ERROR_INODE(inode,
1717 "ex %p == NULL or eh %p == NULL", ex, eh);
1718 return -EFSCORRUPTED;
1719 }
1720
1721 if (depth == 0) {
1722 /* there is no tree at all */
1723 return 0;
1724 }
1725
1726 if (ex != EXT_FIRST_EXTENT(eh)) {
1727 /* we correct tree if first leaf got modified only */
1728 return 0;
1729 }
1730
1731 /*
1732 * TODO: we need correction if border is smaller than current one
1733 */
1734 k = depth - 1;
1735 border = path[depth].p_ext->ee_block;
1736 err = ext4_ext_get_access(handle, inode, path: path + k);
1737 if (err)
1738 return err;
1739 path[k].p_idx->ei_block = border;
1740 err = ext4_ext_dirty(handle, inode, path + k);
1741 if (err)
1742 return err;
1743
1744 while (k--) {
1745 /* change all left-side indexes */
1746 if (path[k+1].p_idx != EXT_FIRST_INDEX(path[k+1].p_hdr))
1747 break;
1748 err = ext4_ext_get_access(handle, inode, path: path + k);
1749 if (err)
1750 goto clean;
1751 path[k].p_idx->ei_block = border;
1752 err = ext4_ext_dirty(handle, inode, path + k);
1753 if (err)
1754 goto clean;
1755 }
1756 return 0;
1757
1758clean:
1759 /*
1760 * The path[k].p_bh is either unmodified or with no verified bit
1761 * set (see ext4_ext_get_access()). So just clear the verified bit
1762 * of the successfully modified extents buffers, which will force
1763 * these extents to be checked to avoid using inconsistent data.
1764 */
1765 while (++k < depth)
1766 clear_buffer_verified(bh: path[k].p_bh);
1767
1768 return err;
1769}
1770
1771static int ext4_can_extents_be_merged(struct inode *inode,
1772 struct ext4_extent *ex1,
1773 struct ext4_extent *ex2)
1774{
1775 unsigned short ext1_ee_len, ext2_ee_len;
1776
1777 if (ext4_ext_is_unwritten(ext: ex1) != ext4_ext_is_unwritten(ext: ex2))
1778 return 0;
1779
1780 ext1_ee_len = ext4_ext_get_actual_len(ext: ex1);
1781 ext2_ee_len = ext4_ext_get_actual_len(ext: ex2);
1782
1783 if (le32_to_cpu(ex1->ee_block) + ext1_ee_len !=
1784 le32_to_cpu(ex2->ee_block))
1785 return 0;
1786
1787 if (ext1_ee_len + ext2_ee_len > EXT_INIT_MAX_LEN)
1788 return 0;
1789
1790 if (ext4_ext_is_unwritten(ext: ex1) &&
1791 ext1_ee_len + ext2_ee_len > EXT_UNWRITTEN_MAX_LEN)
1792 return 0;
1793#ifdef AGGRESSIVE_TEST
1794 if (ext1_ee_len >= 4)
1795 return 0;
1796#endif
1797
1798 if (ext4_ext_pblock(ex: ex1) + ext1_ee_len == ext4_ext_pblock(ex: ex2))
1799 return 1;
1800 return 0;
1801}
1802
1803/*
1804 * This function tries to merge the "ex" extent to the next extent in the tree.
1805 * It always tries to merge towards right. If you want to merge towards
1806 * left, pass "ex - 1" as argument instead of "ex".
1807 * Returns 0 if the extents (ex and ex+1) were _not_ merged and returns
1808 * 1 if they got merged.
1809 */
1810static int ext4_ext_try_to_merge_right(struct inode *inode,
1811 struct ext4_ext_path *path,
1812 struct ext4_extent *ex)
1813{
1814 struct ext4_extent_header *eh;
1815 unsigned int depth, len;
1816 int merge_done = 0, unwritten;
1817
1818 depth = ext_depth(inode);
1819 BUG_ON(path[depth].p_hdr == NULL);
1820 eh = path[depth].p_hdr;
1821
1822 while (ex < EXT_LAST_EXTENT(eh)) {
1823 if (!ext4_can_extents_be_merged(inode, ex1: ex, ex2: ex + 1))
1824 break;
1825 /* merge with next extent! */
1826 unwritten = ext4_ext_is_unwritten(ext: ex);
1827 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
1828 + ext4_ext_get_actual_len(ex + 1));
1829 if (unwritten)
1830 ext4_ext_mark_unwritten(ext: ex);
1831
1832 if (ex + 1 < EXT_LAST_EXTENT(eh)) {
1833 len = (EXT_LAST_EXTENT(eh) - ex - 1)
1834 * sizeof(struct ext4_extent);
1835 memmove(dest: ex + 1, src: ex + 2, count: len);
1836 }
1837 le16_add_cpu(var: &eh->eh_entries, val: -1);
1838 merge_done = 1;
1839 WARN_ON(eh->eh_entries == 0);
1840 if (!eh->eh_entries)
1841 EXT4_ERROR_INODE(inode, "eh->eh_entries = 0!");
1842 }
1843
1844 return merge_done;
1845}
1846
1847/*
1848 * This function does a very simple check to see if we can collapse
1849 * an extent tree with a single extent tree leaf block into the inode.
1850 */
1851static void ext4_ext_try_to_merge_up(handle_t *handle,
1852 struct inode *inode,
1853 struct ext4_ext_path *path)
1854{
1855 size_t s;
1856 unsigned max_root = ext4_ext_space_root(inode, check: 0);
1857 ext4_fsblk_t blk;
1858
1859 if ((path[0].p_depth != 1) ||
1860 (le16_to_cpu(path[0].p_hdr->eh_entries) != 1) ||
1861 (le16_to_cpu(path[1].p_hdr->eh_entries) > max_root))
1862 return;
1863
1864 /*
1865 * We need to modify the block allocation bitmap and the block
1866 * group descriptor to release the extent tree block. If we
1867 * can't get the journal credits, give up.
1868 */
1869 if (ext4_journal_extend(handle, nblocks: 2,
1870 revoke: ext4_free_metadata_revoke_credits(sb: inode->i_sb, blocks: 1)))
1871 return;
1872
1873 /*
1874 * Copy the extent data up to the inode
1875 */
1876 blk = ext4_idx_pblock(ix: path[0].p_idx);
1877 s = le16_to_cpu(path[1].p_hdr->eh_entries) *
1878 sizeof(struct ext4_extent_idx);
1879 s += sizeof(struct ext4_extent_header);
1880
1881 path[1].p_maxdepth = path[0].p_maxdepth;
1882 memcpy(to: path[0].p_hdr, from: path[1].p_hdr, len: s);
1883 path[0].p_depth = 0;
1884 path[0].p_ext = EXT_FIRST_EXTENT(path[0].p_hdr) +
1885 (path[1].p_ext - EXT_FIRST_EXTENT(path[1].p_hdr));
1886 path[0].p_hdr->eh_max = cpu_to_le16(max_root);
1887
1888 ext4_ext_path_brelse(path: path + 1);
1889 ext4_free_blocks(handle, inode, NULL, block: blk, count: 1,
1890 EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET);
1891}
1892
1893/*
1894 * This function tries to merge the @ex extent to neighbours in the tree, then
1895 * tries to collapse the extent tree into the inode.
1896 */
1897static void ext4_ext_try_to_merge(handle_t *handle,
1898 struct inode *inode,
1899 struct ext4_ext_path *path,
1900 struct ext4_extent *ex)
1901{
1902 struct ext4_extent_header *eh;
1903 unsigned int depth;
1904 int merge_done = 0;
1905
1906 depth = ext_depth(inode);
1907 BUG_ON(path[depth].p_hdr == NULL);
1908 eh = path[depth].p_hdr;
1909
1910 if (ex > EXT_FIRST_EXTENT(eh))
1911 merge_done = ext4_ext_try_to_merge_right(inode, path, ex: ex - 1);
1912
1913 if (!merge_done)
1914 (void) ext4_ext_try_to_merge_right(inode, path, ex);
1915
1916 ext4_ext_try_to_merge_up(handle, inode, path);
1917}
1918
1919/*
1920 * check if a portion of the "newext" extent overlaps with an
1921 * existing extent.
1922 *
1923 * If there is an overlap discovered, it updates the length of the newext
1924 * such that there will be no overlap, and then returns 1.
1925 * If there is no overlap found, it returns 0.
1926 */
1927static unsigned int ext4_ext_check_overlap(struct ext4_sb_info *sbi,
1928 struct inode *inode,
1929 struct ext4_extent *newext,
1930 struct ext4_ext_path *path)
1931{
1932 ext4_lblk_t b1, b2;
1933 unsigned int depth, len1;
1934 unsigned int ret = 0;
1935
1936 b1 = le32_to_cpu(newext->ee_block);
1937 len1 = ext4_ext_get_actual_len(ext: newext);
1938 depth = ext_depth(inode);
1939 if (!path[depth].p_ext)
1940 goto out;
1941 b2 = EXT4_LBLK_CMASK(sbi, le32_to_cpu(path[depth].p_ext->ee_block));
1942
1943 /*
1944 * get the next allocated block if the extent in the path
1945 * is before the requested block(s)
1946 */
1947 if (b2 < b1) {
1948 b2 = ext4_ext_next_allocated_block(path);
1949 if (b2 == EXT_MAX_BLOCKS)
1950 goto out;
1951 b2 = EXT4_LBLK_CMASK(sbi, b2);
1952 }
1953
1954 /* check for wrap through zero on extent logical start block*/
1955 if (b1 + len1 < b1) {
1956 len1 = EXT_MAX_BLOCKS - b1;
1957 newext->ee_len = cpu_to_le16(len1);
1958 ret = 1;
1959 }
1960
1961 /* check for overlap */
1962 if (b1 + len1 > b2) {
1963 newext->ee_len = cpu_to_le16(b2 - b1);
1964 ret = 1;
1965 }
1966out:
1967 return ret;
1968}
1969
1970/*
1971 * ext4_ext_insert_extent:
1972 * tries to merge requested extent into the existing extent or
1973 * inserts requested extent as new one into the tree,
1974 * creating new leaf in the no-space case.
1975 */
1976struct ext4_ext_path *
1977ext4_ext_insert_extent(handle_t *handle, struct inode *inode,
1978 struct ext4_ext_path *path,
1979 struct ext4_extent *newext, int gb_flags)
1980{
1981 struct ext4_extent_header *eh;
1982 struct ext4_extent *ex, *fex;
1983 struct ext4_extent *nearex; /* nearest extent */
1984 int depth, len, err = 0;
1985 ext4_lblk_t next;
1986 int mb_flags = 0, unwritten;
1987
1988 if (gb_flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
1989 mb_flags |= EXT4_MB_DELALLOC_RESERVED;
1990 if (unlikely(ext4_ext_get_actual_len(newext) == 0)) {
1991 EXT4_ERROR_INODE(inode, "ext4_ext_get_actual_len(newext) == 0");
1992 err = -EFSCORRUPTED;
1993 goto errout;
1994 }
1995 depth = ext_depth(inode);
1996 ex = path[depth].p_ext;
1997 eh = path[depth].p_hdr;
1998 if (unlikely(path[depth].p_hdr == NULL)) {
1999 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
2000 err = -EFSCORRUPTED;
2001 goto errout;
2002 }
2003
2004 /* try to insert block into found extent and return */
2005 if (ex && !(gb_flags & EXT4_GET_BLOCKS_PRE_IO)) {
2006
2007 /*
2008 * Try to see whether we should rather test the extent on
2009 * right from ex, or from the left of ex. This is because
2010 * ext4_find_extent() can return either extent on the
2011 * left, or on the right from the searched position. This
2012 * will make merging more effective.
2013 */
2014 if (ex < EXT_LAST_EXTENT(eh) &&
2015 (le32_to_cpu(ex->ee_block) +
2016 ext4_ext_get_actual_len(ext: ex) <
2017 le32_to_cpu(newext->ee_block))) {
2018 ex += 1;
2019 goto prepend;
2020 } else if ((ex > EXT_FIRST_EXTENT(eh)) &&
2021 (le32_to_cpu(newext->ee_block) +
2022 ext4_ext_get_actual_len(ext: newext) <
2023 le32_to_cpu(ex->ee_block)))
2024 ex -= 1;
2025
2026 /* Try to append newex to the ex */
2027 if (ext4_can_extents_be_merged(inode, ex1: ex, ex2: newext)) {
2028 ext_debug(inode, "append [%d]%d block to %u:[%d]%d"
2029 "(from %llu)\n",
2030 ext4_ext_is_unwritten(newext),
2031 ext4_ext_get_actual_len(newext),
2032 le32_to_cpu(ex->ee_block),
2033 ext4_ext_is_unwritten(ex),
2034 ext4_ext_get_actual_len(ex),
2035 ext4_ext_pblock(ex));
2036 err = ext4_ext_get_access(handle, inode,
2037 path: path + depth);
2038 if (err)
2039 goto errout;
2040 unwritten = ext4_ext_is_unwritten(ext: ex);
2041 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
2042 + ext4_ext_get_actual_len(newext));
2043 if (unwritten)
2044 ext4_ext_mark_unwritten(ext: ex);
2045 nearex = ex;
2046 goto merge;
2047 }
2048
2049prepend:
2050 /* Try to prepend newex to the ex */
2051 if (ext4_can_extents_be_merged(inode, ex1: newext, ex2: ex)) {
2052 ext_debug(inode, "prepend %u[%d]%d block to %u:[%d]%d"
2053 "(from %llu)\n",
2054 le32_to_cpu(newext->ee_block),
2055 ext4_ext_is_unwritten(newext),
2056 ext4_ext_get_actual_len(newext),
2057 le32_to_cpu(ex->ee_block),
2058 ext4_ext_is_unwritten(ex),
2059 ext4_ext_get_actual_len(ex),
2060 ext4_ext_pblock(ex));
2061 err = ext4_ext_get_access(handle, inode,
2062 path: path + depth);
2063 if (err)
2064 goto errout;
2065
2066 unwritten = ext4_ext_is_unwritten(ext: ex);
2067 ex->ee_block = newext->ee_block;
2068 ext4_ext_store_pblock(ex, pb: ext4_ext_pblock(ex: newext));
2069 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
2070 + ext4_ext_get_actual_len(newext));
2071 if (unwritten)
2072 ext4_ext_mark_unwritten(ext: ex);
2073 nearex = ex;
2074 goto merge;
2075 }
2076 }
2077
2078 depth = ext_depth(inode);
2079 eh = path[depth].p_hdr;
2080 if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max))
2081 goto has_space;
2082
2083 /* probably next leaf has space for us? */
2084 fex = EXT_LAST_EXTENT(eh);
2085 next = EXT_MAX_BLOCKS;
2086 if (le32_to_cpu(newext->ee_block) > le32_to_cpu(fex->ee_block))
2087 next = ext4_ext_next_leaf_block(path);
2088 if (next != EXT_MAX_BLOCKS) {
2089 struct ext4_ext_path *npath;
2090
2091 ext_debug(inode, "next leaf block - %u\n", next);
2092 npath = ext4_find_extent(inode, block: next, NULL, flags: gb_flags);
2093 if (IS_ERR(ptr: npath)) {
2094 err = PTR_ERR(ptr: npath);
2095 goto errout;
2096 }
2097 BUG_ON(npath->p_depth != path->p_depth);
2098 eh = npath[depth].p_hdr;
2099 if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max)) {
2100 ext_debug(inode, "next leaf isn't full(%d)\n",
2101 le16_to_cpu(eh->eh_entries));
2102 ext4_free_ext_path(path);
2103 path = npath;
2104 goto has_space;
2105 }
2106 ext_debug(inode, "next leaf has no free space(%d,%d)\n",
2107 le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max));
2108 ext4_free_ext_path(path: npath);
2109 }
2110
2111 /*
2112 * There is no free space in the found leaf.
2113 * We're gonna add a new leaf in the tree.
2114 */
2115 if (gb_flags & EXT4_GET_BLOCKS_METADATA_NOFAIL)
2116 mb_flags |= EXT4_MB_USE_RESERVED;
2117 path = ext4_ext_create_new_leaf(handle, inode, mb_flags, gb_flags,
2118 path, newext);
2119 if (IS_ERR(ptr: path))
2120 return path;
2121 depth = ext_depth(inode);
2122 eh = path[depth].p_hdr;
2123
2124has_space:
2125 nearex = path[depth].p_ext;
2126
2127 err = ext4_ext_get_access(handle, inode, path: path + depth);
2128 if (err)
2129 goto errout;
2130
2131 if (!nearex) {
2132 /* there is no extent in this leaf, create first one */
2133 ext_debug(inode, "first extent in the leaf: %u:%llu:[%d]%d\n",
2134 le32_to_cpu(newext->ee_block),
2135 ext4_ext_pblock(newext),
2136 ext4_ext_is_unwritten(newext),
2137 ext4_ext_get_actual_len(newext));
2138 nearex = EXT_FIRST_EXTENT(eh);
2139 } else {
2140 if (le32_to_cpu(newext->ee_block)
2141 > le32_to_cpu(nearex->ee_block)) {
2142 /* Insert after */
2143 ext_debug(inode, "insert %u:%llu:[%d]%d before: "
2144 "nearest %p\n",
2145 le32_to_cpu(newext->ee_block),
2146 ext4_ext_pblock(newext),
2147 ext4_ext_is_unwritten(newext),
2148 ext4_ext_get_actual_len(newext),
2149 nearex);
2150 nearex++;
2151 } else {
2152 /* Insert before */
2153 BUG_ON(newext->ee_block == nearex->ee_block);
2154 ext_debug(inode, "insert %u:%llu:[%d]%d after: "
2155 "nearest %p\n",
2156 le32_to_cpu(newext->ee_block),
2157 ext4_ext_pblock(newext),
2158 ext4_ext_is_unwritten(newext),
2159 ext4_ext_get_actual_len(newext),
2160 nearex);
2161 }
2162 len = EXT_LAST_EXTENT(eh) - nearex + 1;
2163 if (len > 0) {
2164 ext_debug(inode, "insert %u:%llu:[%d]%d: "
2165 "move %d extents from 0x%p to 0x%p\n",
2166 le32_to_cpu(newext->ee_block),
2167 ext4_ext_pblock(newext),
2168 ext4_ext_is_unwritten(newext),
2169 ext4_ext_get_actual_len(newext),
2170 len, nearex, nearex + 1);
2171 memmove(dest: nearex + 1, src: nearex,
2172 count: len * sizeof(struct ext4_extent));
2173 }
2174 }
2175
2176 le16_add_cpu(var: &eh->eh_entries, val: 1);
2177 path[depth].p_ext = nearex;
2178 nearex->ee_block = newext->ee_block;
2179 ext4_ext_store_pblock(ex: nearex, pb: ext4_ext_pblock(ex: newext));
2180 nearex->ee_len = newext->ee_len;
2181
2182merge:
2183 /* try to merge extents */
2184 if (!(gb_flags & EXT4_GET_BLOCKS_PRE_IO))
2185 ext4_ext_try_to_merge(handle, inode, path, ex: nearex);
2186
2187 /* time to correct all indexes above */
2188 err = ext4_ext_correct_indexes(handle, inode, path);
2189 if (err)
2190 goto errout;
2191
2192 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
2193 if (err)
2194 goto errout;
2195
2196 return path;
2197
2198errout:
2199 ext4_free_ext_path(path);
2200 return ERR_PTR(error: err);
2201}
2202
2203static int ext4_fill_es_cache_info(struct inode *inode,
2204 ext4_lblk_t block, ext4_lblk_t num,
2205 struct fiemap_extent_info *fieinfo)
2206{
2207 ext4_lblk_t next, end = block + num - 1;
2208 struct extent_status es;
2209 unsigned char blksize_bits = inode->i_sb->s_blocksize_bits;
2210 unsigned int flags;
2211 int err;
2212
2213 while (block <= end) {
2214 next = 0;
2215 flags = 0;
2216 if (!ext4_es_lookup_extent(inode, lblk: block, next_lblk: &next, es: &es))
2217 break;
2218 if (ext4_es_is_unwritten(es: &es))
2219 flags |= FIEMAP_EXTENT_UNWRITTEN;
2220 if (ext4_es_is_delayed(es: &es))
2221 flags |= (FIEMAP_EXTENT_DELALLOC |
2222 FIEMAP_EXTENT_UNKNOWN);
2223 if (ext4_es_is_hole(es: &es))
2224 flags |= EXT4_FIEMAP_EXTENT_HOLE;
2225 if (next == 0)
2226 flags |= FIEMAP_EXTENT_LAST;
2227 if (flags & (FIEMAP_EXTENT_DELALLOC|
2228 EXT4_FIEMAP_EXTENT_HOLE))
2229 es.es_pblk = 0;
2230 else
2231 es.es_pblk = ext4_es_pblock(es: &es);
2232 err = fiemap_fill_next_extent(info: fieinfo,
2233 logical: (__u64)es.es_lblk << blksize_bits,
2234 phys: (__u64)es.es_pblk << blksize_bits,
2235 len: (__u64)es.es_len << blksize_bits,
2236 flags);
2237 if (next == 0)
2238 break;
2239 block = next;
2240 if (err < 0)
2241 return err;
2242 if (err == 1)
2243 return 0;
2244 }
2245 return 0;
2246}
2247
2248
2249/*
2250 * ext4_ext_find_hole - find hole around given block according to the given path
2251 * @inode: inode we lookup in
2252 * @path: path in extent tree to @lblk
2253 * @lblk: pointer to logical block around which we want to determine hole
2254 *
2255 * Determine hole length (and start if easily possible) around given logical
2256 * block. We don't try too hard to find the beginning of the hole but @path
2257 * actually points to extent before @lblk, we provide it.
2258 *
2259 * The function returns the length of a hole starting at @lblk. We update @lblk
2260 * to the beginning of the hole if we managed to find it.
2261 */
2262static ext4_lblk_t ext4_ext_find_hole(struct inode *inode,
2263 struct ext4_ext_path *path,
2264 ext4_lblk_t *lblk)
2265{
2266 int depth = ext_depth(inode);
2267 struct ext4_extent *ex;
2268 ext4_lblk_t len;
2269
2270 ex = path[depth].p_ext;
2271 if (ex == NULL) {
2272 /* there is no extent yet, so gap is [0;-] */
2273 *lblk = 0;
2274 len = EXT_MAX_BLOCKS;
2275 } else if (*lblk < le32_to_cpu(ex->ee_block)) {
2276 len = le32_to_cpu(ex->ee_block) - *lblk;
2277 } else if (*lblk >= le32_to_cpu(ex->ee_block)
2278 + ext4_ext_get_actual_len(ext: ex)) {
2279 ext4_lblk_t next;
2280
2281 *lblk = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ext: ex);
2282 next = ext4_ext_next_allocated_block(path);
2283 BUG_ON(next == *lblk);
2284 len = next - *lblk;
2285 } else {
2286 BUG();
2287 }
2288 return len;
2289}
2290
2291/*
2292 * ext4_ext_rm_idx:
2293 * removes index from the index block.
2294 */
2295static int ext4_ext_rm_idx(handle_t *handle, struct inode *inode,
2296 struct ext4_ext_path *path, int depth)
2297{
2298 int err;
2299 ext4_fsblk_t leaf;
2300 int k = depth - 1;
2301
2302 /* free index block */
2303 leaf = ext4_idx_pblock(ix: path[k].p_idx);
2304 if (unlikely(path[k].p_hdr->eh_entries == 0)) {
2305 EXT4_ERROR_INODE(inode, "path[%d].p_hdr->eh_entries == 0", k);
2306 return -EFSCORRUPTED;
2307 }
2308 err = ext4_ext_get_access(handle, inode, path: path + k);
2309 if (err)
2310 return err;
2311
2312 if (path[k].p_idx != EXT_LAST_INDEX(path[k].p_hdr)) {
2313 int len = EXT_LAST_INDEX(path[k].p_hdr) - path[k].p_idx;
2314 len *= sizeof(struct ext4_extent_idx);
2315 memmove(dest: path[k].p_idx, src: path[k].p_idx + 1, count: len);
2316 }
2317
2318 le16_add_cpu(var: &path[k].p_hdr->eh_entries, val: -1);
2319 err = ext4_ext_dirty(handle, inode, path + k);
2320 if (err)
2321 return err;
2322 ext_debug(inode, "index is empty, remove it, free block %llu\n", leaf);
2323 trace_ext4_ext_rm_idx(inode, pblk: leaf);
2324
2325 ext4_free_blocks(handle, inode, NULL, block: leaf, count: 1,
2326 EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET);
2327
2328 while (--k >= 0) {
2329 if (path[k + 1].p_idx != EXT_FIRST_INDEX(path[k + 1].p_hdr))
2330 break;
2331 err = ext4_ext_get_access(handle, inode, path: path + k);
2332 if (err)
2333 goto clean;
2334 path[k].p_idx->ei_block = path[k + 1].p_idx->ei_block;
2335 err = ext4_ext_dirty(handle, inode, path + k);
2336 if (err)
2337 goto clean;
2338 }
2339 return 0;
2340
2341clean:
2342 /*
2343 * The path[k].p_bh is either unmodified or with no verified bit
2344 * set (see ext4_ext_get_access()). So just clear the verified bit
2345 * of the successfully modified extents buffers, which will force
2346 * these extents to be checked to avoid using inconsistent data.
2347 */
2348 while (++k < depth)
2349 clear_buffer_verified(bh: path[k].p_bh);
2350
2351 return err;
2352}
2353
2354/*
2355 * ext4_ext_calc_credits_for_single_extent:
2356 * This routine returns max. credits that needed to insert an extent
2357 * to the extent tree.
2358 * When pass the actual path, the caller should calculate credits
2359 * under i_data_sem.
2360 */
2361int ext4_ext_calc_credits_for_single_extent(struct inode *inode, int nrblocks,
2362 struct ext4_ext_path *path)
2363{
2364 if (path) {
2365 int depth = ext_depth(inode);
2366 int ret = 0;
2367
2368 /* probably there is space in leaf? */
2369 if (le16_to_cpu(path[depth].p_hdr->eh_entries)
2370 < le16_to_cpu(path[depth].p_hdr->eh_max)) {
2371
2372 /*
2373 * There are some space in the leaf tree, no
2374 * need to account for leaf block credit
2375 *
2376 * bitmaps and block group descriptor blocks
2377 * and other metadata blocks still need to be
2378 * accounted.
2379 */
2380 /* 1 bitmap, 1 block group descriptor */
2381 ret = 2 + EXT4_META_TRANS_BLOCKS(inode->i_sb);
2382 return ret;
2383 }
2384 }
2385
2386 return ext4_chunk_trans_blocks(inode, nrblocks);
2387}
2388
2389/*
2390 * How many index/leaf blocks need to change/allocate to add @extents extents?
2391 *
2392 * If we add a single extent, then in the worse case, each tree level
2393 * index/leaf need to be changed in case of the tree split.
2394 *
2395 * If more extents are inserted, they could cause the whole tree split more
2396 * than once, but this is really rare.
2397 */
2398int ext4_ext_index_trans_blocks(struct inode *inode, int extents)
2399{
2400 int index;
2401
2402 /* If we are converting the inline data, only one is needed here. */
2403 if (ext4_has_inline_data(inode))
2404 return 1;
2405
2406 /*
2407 * Extent tree can change between the time we estimate credits and
2408 * the time we actually modify the tree. Assume the worst case.
2409 */
2410 if (extents <= 1)
2411 index = (EXT4_MAX_EXTENT_DEPTH * 2) + extents;
2412 else
2413 index = (EXT4_MAX_EXTENT_DEPTH * 3) +
2414 DIV_ROUND_UP(extents, ext4_ext_space_block(inode, 0));
2415
2416 return index;
2417}
2418
2419static inline int get_default_free_blocks_flags(struct inode *inode)
2420{
2421 if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode) ||
2422 ext4_test_inode_flag(inode, bit: EXT4_INODE_EA_INODE))
2423 return EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET;
2424 else if (ext4_should_journal_data(inode))
2425 return EXT4_FREE_BLOCKS_FORGET;
2426 return 0;
2427}
2428
2429/*
2430 * ext4_rereserve_cluster - increment the reserved cluster count when
2431 * freeing a cluster with a pending reservation
2432 *
2433 * @inode - file containing the cluster
2434 * @lblk - logical block in cluster to be reserved
2435 *
2436 * Increments the reserved cluster count and adjusts quota in a bigalloc
2437 * file system when freeing a partial cluster containing at least one
2438 * delayed and unwritten block. A partial cluster meeting that
2439 * requirement will have a pending reservation. If so, the
2440 * RERESERVE_CLUSTER flag is used when calling ext4_free_blocks() to
2441 * defer reserved and allocated space accounting to a subsequent call
2442 * to this function.
2443 */
2444static void ext4_rereserve_cluster(struct inode *inode, ext4_lblk_t lblk)
2445{
2446 struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
2447 struct ext4_inode_info *ei = EXT4_I(inode);
2448
2449 dquot_reclaim_block(inode, EXT4_C2B(sbi, 1));
2450
2451 spin_lock(lock: &ei->i_block_reservation_lock);
2452 ei->i_reserved_data_blocks++;
2453 percpu_counter_add(fbc: &sbi->s_dirtyclusters_counter, amount: 1);
2454 spin_unlock(lock: &ei->i_block_reservation_lock);
2455
2456 percpu_counter_add(fbc: &sbi->s_freeclusters_counter, amount: 1);
2457 ext4_remove_pending(inode, lblk);
2458}
2459
2460static int ext4_remove_blocks(handle_t *handle, struct inode *inode,
2461 struct ext4_extent *ex,
2462 struct partial_cluster *partial,
2463 ext4_lblk_t from, ext4_lblk_t to)
2464{
2465 struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
2466 unsigned short ee_len = ext4_ext_get_actual_len(ext: ex);
2467 ext4_fsblk_t last_pblk, pblk;
2468 ext4_lblk_t num;
2469 int flags;
2470
2471 /* only extent tail removal is allowed */
2472 if (from < le32_to_cpu(ex->ee_block) ||
2473 to != le32_to_cpu(ex->ee_block) + ee_len - 1) {
2474 ext4_error(sbi->s_sb,
2475 "strange request: removal(2) %u-%u from %u:%u",
2476 from, to, le32_to_cpu(ex->ee_block), ee_len);
2477 return 0;
2478 }
2479
2480#ifdef EXTENTS_STATS
2481 spin_lock(&sbi->s_ext_stats_lock);
2482 sbi->s_ext_blocks += ee_len;
2483 sbi->s_ext_extents++;
2484 if (ee_len < sbi->s_ext_min)
2485 sbi->s_ext_min = ee_len;
2486 if (ee_len > sbi->s_ext_max)
2487 sbi->s_ext_max = ee_len;
2488 if (ext_depth(inode) > sbi->s_depth_max)
2489 sbi->s_depth_max = ext_depth(inode);
2490 spin_unlock(&sbi->s_ext_stats_lock);
2491#endif
2492
2493 trace_ext4_remove_blocks(inode, ex, from, to, pc: partial);
2494
2495 /*
2496 * if we have a partial cluster, and it's different from the
2497 * cluster of the last block in the extent, we free it
2498 */
2499 last_pblk = ext4_ext_pblock(ex) + ee_len - 1;
2500
2501 if (partial->state != initial &&
2502 partial->pclu != EXT4_B2C(sbi, last_pblk)) {
2503 if (partial->state == tofree) {
2504 flags = get_default_free_blocks_flags(inode);
2505 if (ext4_is_pending(inode, lblk: partial->lblk))
2506 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
2507 ext4_free_blocks(handle, inode, NULL,
2508 EXT4_C2B(sbi, partial->pclu),
2509 count: sbi->s_cluster_ratio, flags);
2510 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
2511 ext4_rereserve_cluster(inode, lblk: partial->lblk);
2512 }
2513 partial->state = initial;
2514 }
2515
2516 num = le32_to_cpu(ex->ee_block) + ee_len - from;
2517 pblk = ext4_ext_pblock(ex) + ee_len - num;
2518
2519 /*
2520 * We free the partial cluster at the end of the extent (if any),
2521 * unless the cluster is used by another extent (partial_cluster
2522 * state is nofree). If a partial cluster exists here, it must be
2523 * shared with the last block in the extent.
2524 */
2525 flags = get_default_free_blocks_flags(inode);
2526
2527 /* partial, left end cluster aligned, right end unaligned */
2528 if ((EXT4_LBLK_COFF(sbi, to) != sbi->s_cluster_ratio - 1) &&
2529 (EXT4_LBLK_CMASK(sbi, to) >= from) &&
2530 (partial->state != nofree)) {
2531 if (ext4_is_pending(inode, lblk: to))
2532 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
2533 ext4_free_blocks(handle, inode, NULL,
2534 EXT4_PBLK_CMASK(sbi, last_pblk),
2535 count: sbi->s_cluster_ratio, flags);
2536 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
2537 ext4_rereserve_cluster(inode, lblk: to);
2538 partial->state = initial;
2539 flags = get_default_free_blocks_flags(inode);
2540 }
2541
2542 flags |= EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER;
2543
2544 /*
2545 * For bigalloc file systems, we never free a partial cluster
2546 * at the beginning of the extent. Instead, we check to see if we
2547 * need to free it on a subsequent call to ext4_remove_blocks,
2548 * or at the end of ext4_ext_rm_leaf or ext4_ext_remove_space.
2549 */
2550 flags |= EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER;
2551 ext4_free_blocks(handle, inode, NULL, block: pblk, count: num, flags);
2552
2553 /* reset the partial cluster if we've freed past it */
2554 if (partial->state != initial && partial->pclu != EXT4_B2C(sbi, pblk))
2555 partial->state = initial;
2556
2557 /*
2558 * If we've freed the entire extent but the beginning is not left
2559 * cluster aligned and is not marked as ineligible for freeing we
2560 * record the partial cluster at the beginning of the extent. It
2561 * wasn't freed by the preceding ext4_free_blocks() call, and we
2562 * need to look farther to the left to determine if it's to be freed
2563 * (not shared with another extent). Else, reset the partial
2564 * cluster - we're either done freeing or the beginning of the
2565 * extent is left cluster aligned.
2566 */
2567 if (EXT4_LBLK_COFF(sbi, from) && num == ee_len) {
2568 if (partial->state == initial) {
2569 partial->pclu = EXT4_B2C(sbi, pblk);
2570 partial->lblk = from;
2571 partial->state = tofree;
2572 }
2573 } else {
2574 partial->state = initial;
2575 }
2576
2577 return 0;
2578}
2579
2580/*
2581 * ext4_ext_rm_leaf() Removes the extents associated with the
2582 * blocks appearing between "start" and "end". Both "start"
2583 * and "end" must appear in the same extent or EIO is returned.
2584 *
2585 * @handle: The journal handle
2586 * @inode: The files inode
2587 * @path: The path to the leaf
2588 * @partial_cluster: The cluster which we'll have to free if all extents
2589 * has been released from it. However, if this value is
2590 * negative, it's a cluster just to the right of the
2591 * punched region and it must not be freed.
2592 * @start: The first block to remove
2593 * @end: The last block to remove
2594 */
2595static int
2596ext4_ext_rm_leaf(handle_t *handle, struct inode *inode,
2597 struct ext4_ext_path *path,
2598 struct partial_cluster *partial,
2599 ext4_lblk_t start, ext4_lblk_t end)
2600{
2601 struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
2602 int err = 0, correct_index = 0;
2603 int depth = ext_depth(inode), credits, revoke_credits;
2604 struct ext4_extent_header *eh;
2605 ext4_lblk_t a, b;
2606 unsigned num;
2607 ext4_lblk_t ex_ee_block;
2608 unsigned short ex_ee_len;
2609 unsigned unwritten = 0;
2610 struct ext4_extent *ex;
2611 ext4_fsblk_t pblk;
2612
2613 /* the header must be checked already in ext4_ext_remove_space() */
2614 ext_debug(inode, "truncate since %u in leaf to %u\n", start, end);
2615 if (!path[depth].p_hdr)
2616 path[depth].p_hdr = ext_block_hdr(bh: path[depth].p_bh);
2617 eh = path[depth].p_hdr;
2618 if (unlikely(path[depth].p_hdr == NULL)) {
2619 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
2620 return -EFSCORRUPTED;
2621 }
2622 /* find where to start removing */
2623 ex = path[depth].p_ext;
2624 if (!ex)
2625 ex = EXT_LAST_EXTENT(eh);
2626
2627 ex_ee_block = le32_to_cpu(ex->ee_block);
2628 ex_ee_len = ext4_ext_get_actual_len(ext: ex);
2629
2630 trace_ext4_ext_rm_leaf(inode, start, ex, pc: partial);
2631
2632 while (ex >= EXT_FIRST_EXTENT(eh) &&
2633 ex_ee_block + ex_ee_len > start) {
2634
2635 if (ext4_ext_is_unwritten(ext: ex))
2636 unwritten = 1;
2637 else
2638 unwritten = 0;
2639
2640 ext_debug(inode, "remove ext %u:[%d]%d\n", ex_ee_block,
2641 unwritten, ex_ee_len);
2642 path[depth].p_ext = ex;
2643
2644 a = max(ex_ee_block, start);
2645 b = min(ex_ee_block + ex_ee_len - 1, end);
2646
2647 ext_debug(inode, " border %u:%u\n", a, b);
2648
2649 /* If this extent is beyond the end of the hole, skip it */
2650 if (end < ex_ee_block) {
2651 /*
2652 * We're going to skip this extent and move to another,
2653 * so note that its first cluster is in use to avoid
2654 * freeing it when removing blocks. Eventually, the
2655 * right edge of the truncated/punched region will
2656 * be just to the left.
2657 */
2658 if (sbi->s_cluster_ratio > 1) {
2659 pblk = ext4_ext_pblock(ex);
2660 partial->pclu = EXT4_B2C(sbi, pblk);
2661 partial->state = nofree;
2662 }
2663 ex--;
2664 ex_ee_block = le32_to_cpu(ex->ee_block);
2665 ex_ee_len = ext4_ext_get_actual_len(ext: ex);
2666 continue;
2667 } else if (b != ex_ee_block + ex_ee_len - 1) {
2668 EXT4_ERROR_INODE(inode,
2669 "can not handle truncate %u:%u "
2670 "on extent %u:%u",
2671 start, end, ex_ee_block,
2672 ex_ee_block + ex_ee_len - 1);
2673 err = -EFSCORRUPTED;
2674 goto out;
2675 } else if (a != ex_ee_block) {
2676 /* remove tail of the extent */
2677 num = a - ex_ee_block;
2678 } else {
2679 /* remove whole extent: excellent! */
2680 num = 0;
2681 }
2682 /*
2683 * 3 for leaf, sb, and inode plus 2 (bmap and group
2684 * descriptor) for each block group; assume two block
2685 * groups plus ex_ee_len/blocks_per_block_group for
2686 * the worst case
2687 */
2688 credits = 7 + 2*(ex_ee_len/EXT4_BLOCKS_PER_GROUP(inode->i_sb));
2689 if (ex == EXT_FIRST_EXTENT(eh)) {
2690 correct_index = 1;
2691 credits += (ext_depth(inode)) + 1;
2692 }
2693 credits += EXT4_MAXQUOTAS_TRANS_BLOCKS(inode->i_sb);
2694 /*
2695 * We may end up freeing some index blocks and data from the
2696 * punched range. Note that partial clusters are accounted for
2697 * by ext4_free_data_revoke_credits().
2698 */
2699 revoke_credits =
2700 ext4_free_metadata_revoke_credits(sb: inode->i_sb,
2701 blocks: ext_depth(inode)) +
2702 ext4_free_data_revoke_credits(inode, blocks: b - a + 1);
2703
2704 err = ext4_datasem_ensure_credits(handle, inode, check_cred: credits,
2705 restart_cred: credits, revoke_cred: revoke_credits);
2706 if (err) {
2707 if (err > 0)
2708 err = -EAGAIN;
2709 goto out;
2710 }
2711
2712 err = ext4_ext_get_access(handle, inode, path: path + depth);
2713 if (err)
2714 goto out;
2715
2716 err = ext4_remove_blocks(handle, inode, ex, partial, from: a, to: b);
2717 if (err)
2718 goto out;
2719
2720 if (num == 0)
2721 /* this extent is removed; mark slot entirely unused */
2722 ext4_ext_store_pblock(ex, pb: 0);
2723
2724 ex->ee_len = cpu_to_le16(num);
2725 /*
2726 * Do not mark unwritten if all the blocks in the
2727 * extent have been removed.
2728 */
2729 if (unwritten && num)
2730 ext4_ext_mark_unwritten(ext: ex);
2731 /*
2732 * If the extent was completely released,
2733 * we need to remove it from the leaf
2734 */
2735 if (num == 0) {
2736 if (end != EXT_MAX_BLOCKS - 1) {
2737 /*
2738 * For hole punching, we need to scoot all the
2739 * extents up when an extent is removed so that
2740 * we dont have blank extents in the middle
2741 */
2742 memmove(dest: ex, src: ex+1, count: (EXT_LAST_EXTENT(eh) - ex) *
2743 sizeof(struct ext4_extent));
2744
2745 /* Now get rid of the one at the end */
2746 memset(EXT_LAST_EXTENT(eh), c: 0,
2747 n: sizeof(struct ext4_extent));
2748 }
2749 le16_add_cpu(var: &eh->eh_entries, val: -1);
2750 }
2751
2752 err = ext4_ext_dirty(handle, inode, path + depth);
2753 if (err)
2754 goto out;
2755
2756 ext_debug(inode, "new extent: %u:%u:%llu\n", ex_ee_block, num,
2757 ext4_ext_pblock(ex));
2758 ex--;
2759 ex_ee_block = le32_to_cpu(ex->ee_block);
2760 ex_ee_len = ext4_ext_get_actual_len(ext: ex);
2761 }
2762
2763 if (correct_index && eh->eh_entries)
2764 err = ext4_ext_correct_indexes(handle, inode, path);
2765
2766 /*
2767 * If there's a partial cluster and at least one extent remains in
2768 * the leaf, free the partial cluster if it isn't shared with the
2769 * current extent. If it is shared with the current extent
2770 * we reset the partial cluster because we've reached the start of the
2771 * truncated/punched region and we're done removing blocks.
2772 */
2773 if (partial->state == tofree && ex >= EXT_FIRST_EXTENT(eh)) {
2774 pblk = ext4_ext_pblock(ex) + ex_ee_len - 1;
2775 if (partial->pclu != EXT4_B2C(sbi, pblk)) {
2776 int flags = get_default_free_blocks_flags(inode);
2777
2778 if (ext4_is_pending(inode, lblk: partial->lblk))
2779 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
2780 ext4_free_blocks(handle, inode, NULL,
2781 EXT4_C2B(sbi, partial->pclu),
2782 count: sbi->s_cluster_ratio, flags);
2783 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
2784 ext4_rereserve_cluster(inode, lblk: partial->lblk);
2785 }
2786 partial->state = initial;
2787 }
2788
2789 /* if this leaf is free, then we should
2790 * remove it from index block above */
2791 if (err == 0 && eh->eh_entries == 0 && path[depth].p_bh != NULL)
2792 err = ext4_ext_rm_idx(handle, inode, path, depth);
2793
2794out:
2795 return err;
2796}
2797
2798/*
2799 * ext4_ext_more_to_rm:
2800 * returns 1 if current index has to be freed (even partial)
2801 */
2802static int
2803ext4_ext_more_to_rm(struct ext4_ext_path *path)
2804{
2805 BUG_ON(path->p_idx == NULL);
2806
2807 if (path->p_idx < EXT_FIRST_INDEX(path->p_hdr))
2808 return 0;
2809
2810 /*
2811 * if truncate on deeper level happened, it wasn't partial,
2812 * so we have to consider current index for truncation
2813 */
2814 if (le16_to_cpu(path->p_hdr->eh_entries) == path->p_block)
2815 return 0;
2816 return 1;
2817}
2818
2819int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start,
2820 ext4_lblk_t end)
2821{
2822 struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
2823 int depth = ext_depth(inode);
2824 struct ext4_ext_path *path = NULL;
2825 struct partial_cluster partial;
2826 handle_t *handle;
2827 int i = 0, err = 0;
2828 int flags = EXT4_EX_NOCACHE | EXT4_EX_NOFAIL;
2829
2830 partial.pclu = 0;
2831 partial.lblk = 0;
2832 partial.state = initial;
2833
2834 ext_debug(inode, "truncate since %u to %u\n", start, end);
2835
2836 /* probably first extent we're gonna free will be last in block */
2837 handle = ext4_journal_start_with_revoke(inode, EXT4_HT_TRUNCATE,
2838 depth + 1,
2839 ext4_free_metadata_revoke_credits(inode->i_sb, depth));
2840 if (IS_ERR(ptr: handle))
2841 return PTR_ERR(ptr: handle);
2842
2843again:
2844 trace_ext4_ext_remove_space(inode, start, end, depth);
2845
2846 /*
2847 * Check if we are removing extents inside the extent tree. If that
2848 * is the case, we are going to punch a hole inside the extent tree
2849 * so we have to check whether we need to split the extent covering
2850 * the last block to remove so we can easily remove the part of it
2851 * in ext4_ext_rm_leaf().
2852 */
2853 if (end < EXT_MAX_BLOCKS - 1) {
2854 struct ext4_extent *ex;
2855 ext4_lblk_t ee_block, ex_end, lblk;
2856 ext4_fsblk_t pblk;
2857
2858 /* find extent for or closest extent to this block */
2859 path = ext4_find_extent(inode, block: end, NULL, flags);
2860 if (IS_ERR(ptr: path)) {
2861 ext4_journal_stop(handle);
2862 return PTR_ERR(ptr: path);
2863 }
2864 depth = ext_depth(inode);
2865 /* Leaf not may not exist only if inode has no blocks at all */
2866 ex = path[depth].p_ext;
2867 if (!ex) {
2868 if (depth) {
2869 EXT4_ERROR_INODE(inode,
2870 "path[%d].p_hdr == NULL",
2871 depth);
2872 err = -EFSCORRUPTED;
2873 }
2874 goto out;
2875 }
2876
2877 ee_block = le32_to_cpu(ex->ee_block);
2878 ex_end = ee_block + ext4_ext_get_actual_len(ext: ex) - 1;
2879
2880 /*
2881 * See if the last block is inside the extent, if so split
2882 * the extent at 'end' block so we can easily remove the
2883 * tail of the first part of the split extent in
2884 * ext4_ext_rm_leaf().
2885 */
2886 if (end >= ee_block && end < ex_end) {
2887
2888 /*
2889 * If we're going to split the extent, note that
2890 * the cluster containing the block after 'end' is
2891 * in use to avoid freeing it when removing blocks.
2892 */
2893 if (sbi->s_cluster_ratio > 1) {
2894 pblk = ext4_ext_pblock(ex) + end - ee_block + 1;
2895 partial.pclu = EXT4_B2C(sbi, pblk);
2896 partial.state = nofree;
2897 }
2898
2899 /*
2900 * Split the extent in two so that 'end' is the last
2901 * block in the first new extent. Also we should not
2902 * fail removing space due to ENOSPC so try to use
2903 * reserved block if that happens.
2904 */
2905 path = ext4_force_split_extent_at(handle, inode, path,
2906 lblk: end + 1, nofail: 1);
2907 if (IS_ERR(ptr: path)) {
2908 err = PTR_ERR(ptr: path);
2909 goto out;
2910 }
2911 } else if (sbi->s_cluster_ratio > 1 && end >= ex_end &&
2912 partial.state == initial) {
2913 /*
2914 * If we're punching, there's an extent to the right.
2915 * If the partial cluster hasn't been set, set it to
2916 * that extent's first cluster and its state to nofree
2917 * so it won't be freed should it contain blocks to be
2918 * removed. If it's already set (tofree/nofree), we're
2919 * retrying and keep the original partial cluster info
2920 * so a cluster marked tofree as a result of earlier
2921 * extent removal is not lost.
2922 */
2923 lblk = ex_end + 1;
2924 err = ext4_ext_search_right(inode, path, logical: &lblk, phys: &pblk,
2925 NULL, flags);
2926 if (err < 0)
2927 goto out;
2928 if (pblk) {
2929 partial.pclu = EXT4_B2C(sbi, pblk);
2930 partial.state = nofree;
2931 }
2932 }
2933 }
2934 /*
2935 * We start scanning from right side, freeing all the blocks
2936 * after i_size and walking into the tree depth-wise.
2937 */
2938 depth = ext_depth(inode);
2939 if (path) {
2940 int k = i = depth;
2941 while (--k > 0)
2942 path[k].p_block =
2943 le16_to_cpu(path[k].p_hdr->eh_entries)+1;
2944 } else {
2945 path = kcalloc(depth + 1, sizeof(struct ext4_ext_path),
2946 GFP_NOFS | __GFP_NOFAIL);
2947 if (path == NULL) {
2948 ext4_journal_stop(handle);
2949 return -ENOMEM;
2950 }
2951 path[0].p_maxdepth = path[0].p_depth = depth;
2952 path[0].p_hdr = ext_inode_hdr(inode);
2953 i = 0;
2954
2955 if (ext4_ext_check(inode, path[0].p_hdr, depth, 0)) {
2956 err = -EFSCORRUPTED;
2957 goto out;
2958 }
2959 }
2960 err = 0;
2961
2962 while (i >= 0 && err == 0) {
2963 if (i == depth) {
2964 /* this is leaf block */
2965 err = ext4_ext_rm_leaf(handle, inode, path,
2966 partial: &partial, start, end);
2967 /* root level has p_bh == NULL, brelse() eats this */
2968 ext4_ext_path_brelse(path: path + i);
2969 i--;
2970 continue;
2971 }
2972
2973 /* this is index block */
2974 if (!path[i].p_hdr) {
2975 ext_debug(inode, "initialize header\n");
2976 path[i].p_hdr = ext_block_hdr(bh: path[i].p_bh);
2977 }
2978
2979 if (!path[i].p_idx) {
2980 /* this level hasn't been touched yet */
2981 path[i].p_idx = EXT_LAST_INDEX(path[i].p_hdr);
2982 path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries)+1;
2983 ext_debug(inode, "init index ptr: hdr 0x%p, num %d\n",
2984 path[i].p_hdr,
2985 le16_to_cpu(path[i].p_hdr->eh_entries));
2986 } else {
2987 /* we were already here, see at next index */
2988 path[i].p_idx--;
2989 }
2990
2991 ext_debug(inode, "level %d - index, first 0x%p, cur 0x%p\n",
2992 i, EXT_FIRST_INDEX(path[i].p_hdr),
2993 path[i].p_idx);
2994 if (ext4_ext_more_to_rm(path: path + i)) {
2995 struct buffer_head *bh;
2996 /* go to the next level */
2997 ext_debug(inode, "move to level %d (block %llu)\n",
2998 i + 1, ext4_idx_pblock(path[i].p_idx));
2999 memset(s: path + i + 1, c: 0, n: sizeof(*path));
3000 bh = read_extent_tree_block(inode, path[i].p_idx,
3001 depth - i - 1, flags);
3002 if (IS_ERR(ptr: bh)) {
3003 /* should we reset i_size? */
3004 err = PTR_ERR(ptr: bh);
3005 break;
3006 }
3007 /* Yield here to deal with large extent trees.
3008 * Should be a no-op if we did IO above. */
3009 cond_resched();
3010 if (WARN_ON(i + 1 > depth)) {
3011 err = -EFSCORRUPTED;
3012 break;
3013 }
3014 path[i + 1].p_bh = bh;
3015
3016 /* save actual number of indexes since this
3017 * number is changed at the next iteration */
3018 path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries);
3019 i++;
3020 } else {
3021 /* we finished processing this index, go up */
3022 if (path[i].p_hdr->eh_entries == 0 && i > 0) {
3023 /* index is empty, remove it;
3024 * handle must be already prepared by the
3025 * truncatei_leaf() */
3026 err = ext4_ext_rm_idx(handle, inode, path, depth: i);
3027 }
3028 /* root level has p_bh == NULL, brelse() eats this */
3029 ext4_ext_path_brelse(path: path + i);
3030 i--;
3031 ext_debug(inode, "return to level %d\n", i);
3032 }
3033 }
3034
3035 trace_ext4_ext_remove_space_done(inode, start, end, depth, pc: &partial,
3036 eh_entries: path->p_hdr->eh_entries);
3037
3038 /*
3039 * if there's a partial cluster and we have removed the first extent
3040 * in the file, then we also free the partial cluster, if any
3041 */
3042 if (partial.state == tofree && err == 0) {
3043 int flags = get_default_free_blocks_flags(inode);
3044
3045 if (ext4_is_pending(inode, lblk: partial.lblk))
3046 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
3047 ext4_free_blocks(handle, inode, NULL,
3048 EXT4_C2B(sbi, partial.pclu),
3049 count: sbi->s_cluster_ratio, flags);
3050 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
3051 ext4_rereserve_cluster(inode, lblk: partial.lblk);
3052 partial.state = initial;
3053 }
3054
3055 /* TODO: flexible tree reduction should be here */
3056 if (path->p_hdr->eh_entries == 0) {
3057 /*
3058 * truncate to zero freed all the tree,
3059 * so we need to correct eh_depth
3060 */
3061 err = ext4_ext_get_access(handle, inode, path);
3062 if (err == 0) {
3063 ext_inode_hdr(inode)->eh_depth = 0;
3064 ext_inode_hdr(inode)->eh_max =
3065 cpu_to_le16(ext4_ext_space_root(inode, 0));
3066 err = ext4_ext_dirty(handle, inode, path);
3067 }
3068 }
3069out:
3070 ext4_free_ext_path(path);
3071 path = NULL;
3072 if (err == -EAGAIN)
3073 goto again;
3074 ext4_journal_stop(handle);
3075
3076 return err;
3077}
3078
3079/*
3080 * called at mount time
3081 */
3082void ext4_ext_init(struct super_block *sb)
3083{
3084 /*
3085 * possible initialization would be here
3086 */
3087
3088 if (ext4_has_feature_extents(sb)) {
3089#if defined(AGGRESSIVE_TEST) || defined(CHECK_BINSEARCH) || defined(EXTENTS_STATS)
3090 printk(KERN_INFO "EXT4-fs: file extents enabled"
3091#ifdef AGGRESSIVE_TEST
3092 ", aggressive tests"
3093#endif
3094#ifdef CHECK_BINSEARCH
3095 ", check binsearch"
3096#endif
3097#ifdef EXTENTS_STATS
3098 ", stats"
3099#endif
3100 "\n");
3101#endif
3102#ifdef EXTENTS_STATS
3103 spin_lock_init(&EXT4_SB(sb)->s_ext_stats_lock);
3104 EXT4_SB(sb)->s_ext_min = 1 << 30;
3105 EXT4_SB(sb)->s_ext_max = 0;
3106#endif
3107 }
3108}
3109
3110/*
3111 * called at umount time
3112 */
3113void ext4_ext_release(struct super_block *sb)
3114{
3115 if (!ext4_has_feature_extents(sb))
3116 return;
3117
3118#ifdef EXTENTS_STATS
3119 if (EXT4_SB(sb)->s_ext_blocks && EXT4_SB(sb)->s_ext_extents) {
3120 struct ext4_sb_info *sbi = EXT4_SB(sb);
3121 printk(KERN_ERR "EXT4-fs: %lu blocks in %lu extents (%lu ave)\n",
3122 sbi->s_ext_blocks, sbi->s_ext_extents,
3123 sbi->s_ext_blocks / sbi->s_ext_extents);
3124 printk(KERN_ERR "EXT4-fs: extents: %lu min, %lu max, max depth %lu\n",
3125 sbi->s_ext_min, sbi->s_ext_max, sbi->s_depth_max);
3126 }
3127#endif
3128}
3129
3130static void ext4_zeroout_es(struct inode *inode, struct ext4_extent *ex)
3131{
3132 ext4_lblk_t ee_block;
3133 ext4_fsblk_t ee_pblock;
3134 unsigned int ee_len;
3135
3136 ee_block = le32_to_cpu(ex->ee_block);
3137 ee_len = ext4_ext_get_actual_len(ext: ex);
3138 ee_pblock = ext4_ext_pblock(ex);
3139
3140 if (ee_len == 0)
3141 return;
3142
3143 ext4_es_insert_extent(inode, lblk: ee_block, len: ee_len, pblk: ee_pblock,
3144 EXTENT_STATUS_WRITTEN, delalloc_reserve_used: false);
3145}
3146
3147/* FIXME!! we need to try to merge to left or right after zero-out */
3148static int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex)
3149{
3150 ext4_fsblk_t ee_pblock;
3151 unsigned int ee_len;
3152
3153 ee_len = ext4_ext_get_actual_len(ext: ex);
3154 ee_pblock = ext4_ext_pblock(ex);
3155 return ext4_issue_zeroout(inode, le32_to_cpu(ex->ee_block), pblk: ee_pblock,
3156 len: ee_len);
3157}
3158
3159/*
3160 * ext4_split_extent_at() splits an extent at given block.
3161 *
3162 * @handle: the journal handle
3163 * @inode: the file inode
3164 * @path: the path to the extent
3165 * @split: the logical block where the extent is splitted.
3166 * @split_flags: indicates if the extent could be zeroout if split fails, and
3167 * the states(init or unwritten) of new extents.
3168 * @flags: flags used to insert new extent to extent tree.
3169 *
3170 *
3171 * Splits extent [a, b] into two extents [a, @split) and [@split, b], states
3172 * of which are determined by split_flag.
3173 *
3174 * There are two cases:
3175 * a> the extent are splitted into two extent.
3176 * b> split is not needed, and just mark the extent.
3177 *
3178 * Return an extent path pointer on success, or an error pointer on failure.
3179 */
3180static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle,
3181 struct inode *inode,
3182 struct ext4_ext_path *path,
3183 ext4_lblk_t split,
3184 int split_flag, int flags)
3185{
3186 ext4_fsblk_t newblock;
3187 ext4_lblk_t ee_block;
3188 struct ext4_extent *ex, newex, orig_ex, zero_ex;
3189 struct ext4_extent *ex2 = NULL;
3190 unsigned int ee_len, depth;
3191 int err = 0;
3192
3193 BUG_ON((split_flag & (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)) ==
3194 (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2));
3195
3196 ext_debug(inode, "logical block %llu\n", (unsigned long long)split);
3197
3198 ext4_ext_show_leaf(inode, path);
3199
3200 depth = ext_depth(inode);
3201 ex = path[depth].p_ext;
3202 ee_block = le32_to_cpu(ex->ee_block);
3203 ee_len = ext4_ext_get_actual_len(ext: ex);
3204 newblock = split - ee_block + ext4_ext_pblock(ex);
3205
3206 BUG_ON(split < ee_block || split >= (ee_block + ee_len));
3207 BUG_ON(!ext4_ext_is_unwritten(ex) &&
3208 split_flag & (EXT4_EXT_MAY_ZEROOUT |
3209 EXT4_EXT_MARK_UNWRIT1 |
3210 EXT4_EXT_MARK_UNWRIT2));
3211
3212 err = ext4_ext_get_access(handle, inode, path: path + depth);
3213 if (err)
3214 goto out;
3215
3216 if (split == ee_block) {
3217 /*
3218 * case b: block @split is the block that the extent begins with
3219 * then we just change the state of the extent, and splitting
3220 * is not needed.
3221 */
3222 if (split_flag & EXT4_EXT_MARK_UNWRIT2)
3223 ext4_ext_mark_unwritten(ext: ex);
3224 else
3225 ext4_ext_mark_initialized(ext: ex);
3226
3227 if (!(flags & EXT4_GET_BLOCKS_PRE_IO))
3228 ext4_ext_try_to_merge(handle, inode, path, ex);
3229
3230 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3231 goto out;
3232 }
3233
3234 /* case a */
3235 memcpy(to: &orig_ex, from: ex, len: sizeof(orig_ex));
3236 ex->ee_len = cpu_to_le16(split - ee_block);
3237 if (split_flag & EXT4_EXT_MARK_UNWRIT1)
3238 ext4_ext_mark_unwritten(ext: ex);
3239
3240 /*
3241 * path may lead to new leaf, not to original leaf any more
3242 * after ext4_ext_insert_extent() returns,
3243 */
3244 err = ext4_ext_dirty(handle, inode, path + depth);
3245 if (err)
3246 goto fix_extent_len;
3247
3248 ex2 = &newex;
3249 ex2->ee_block = cpu_to_le32(split);
3250 ex2->ee_len = cpu_to_le16(ee_len - (split - ee_block));
3251 ext4_ext_store_pblock(ex: ex2, pb: newblock);
3252 if (split_flag & EXT4_EXT_MARK_UNWRIT2)
3253 ext4_ext_mark_unwritten(ext: ex2);
3254
3255 path = ext4_ext_insert_extent(handle, inode, path, newext: &newex, gb_flags: flags);
3256 if (!IS_ERR(ptr: path))
3257 goto out;
3258
3259 err = PTR_ERR(ptr: path);
3260 if (err != -ENOSPC && err != -EDQUOT && err != -ENOMEM)
3261 return path;
3262
3263 /*
3264 * Get a new path to try to zeroout or fix the extent length.
3265 * Using EXT4_EX_NOFAIL guarantees that ext4_find_extent()
3266 * will not return -ENOMEM, otherwise -ENOMEM will cause a
3267 * retry in do_writepages(), and a WARN_ON may be triggered
3268 * in ext4_da_update_reserve_space() due to an incorrect
3269 * ee_len causing the i_reserved_data_blocks exception.
3270 */
3271 path = ext4_find_extent(inode, block: ee_block, NULL, flags: flags | EXT4_EX_NOFAIL);
3272 if (IS_ERR(ptr: path)) {
3273 EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %ld",
3274 split, PTR_ERR(path));
3275 return path;
3276 }
3277 depth = ext_depth(inode);
3278 ex = path[depth].p_ext;
3279
3280 if (EXT4_EXT_MAY_ZEROOUT & split_flag) {
3281 if (split_flag & (EXT4_EXT_DATA_VALID1|EXT4_EXT_DATA_VALID2)) {
3282 if (split_flag & EXT4_EXT_DATA_VALID1) {
3283 err = ext4_ext_zeroout(inode, ex: ex2);
3284 zero_ex.ee_block = ex2->ee_block;
3285 zero_ex.ee_len = cpu_to_le16(
3286 ext4_ext_get_actual_len(ex2));
3287 ext4_ext_store_pblock(ex: &zero_ex,
3288 pb: ext4_ext_pblock(ex: ex2));
3289 } else {
3290 err = ext4_ext_zeroout(inode, ex);
3291 zero_ex.ee_block = ex->ee_block;
3292 zero_ex.ee_len = cpu_to_le16(
3293 ext4_ext_get_actual_len(ex));
3294 ext4_ext_store_pblock(ex: &zero_ex,
3295 pb: ext4_ext_pblock(ex));
3296 }
3297 } else {
3298 err = ext4_ext_zeroout(inode, ex: &orig_ex);
3299 zero_ex.ee_block = orig_ex.ee_block;
3300 zero_ex.ee_len = cpu_to_le16(
3301 ext4_ext_get_actual_len(&orig_ex));
3302 ext4_ext_store_pblock(ex: &zero_ex,
3303 pb: ext4_ext_pblock(ex: &orig_ex));
3304 }
3305
3306 if (!err) {
3307 /* update the extent length and mark as initialized */
3308 ex->ee_len = cpu_to_le16(ee_len);
3309 ext4_ext_try_to_merge(handle, inode, path, ex);
3310 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3311 if (!err)
3312 /* update extent status tree */
3313 ext4_zeroout_es(inode, ex: &zero_ex);
3314 /* If we failed at this point, we don't know in which
3315 * state the extent tree exactly is so don't try to fix
3316 * length of the original extent as it may do even more
3317 * damage.
3318 */
3319 goto out;
3320 }
3321 }
3322
3323fix_extent_len:
3324 ex->ee_len = orig_ex.ee_len;
3325 /*
3326 * Ignore ext4_ext_dirty return value since we are already in error path
3327 * and err is a non-zero error code.
3328 */
3329 ext4_ext_dirty(handle, inode, path + path->p_depth);
3330out:
3331 if (err) {
3332 ext4_free_ext_path(path);
3333 path = ERR_PTR(error: err);
3334 }
3335 ext4_ext_show_leaf(inode, path);
3336 return path;
3337}
3338
3339/*
3340 * ext4_split_extent() splits an extent and mark extent which is covered
3341 * by @map as split_flags indicates
3342 *
3343 * It may result in splitting the extent into multiple extents (up to three)
3344 * There are three possibilities:
3345 * a> There is no split required
3346 * b> Splits in two extents: Split is happening at either end of the extent
3347 * c> Splits in three extents: Somone is splitting in middle of the extent
3348 *
3349 */
3350static struct ext4_ext_path *ext4_split_extent(handle_t *handle,
3351 struct inode *inode,
3352 struct ext4_ext_path *path,
3353 struct ext4_map_blocks *map,
3354 int split_flag, int flags,
3355 unsigned int *allocated)
3356{
3357 ext4_lblk_t ee_block;
3358 struct ext4_extent *ex;
3359 unsigned int ee_len, depth;
3360 int unwritten;
3361 int split_flag1, flags1;
3362
3363 depth = ext_depth(inode);
3364 ex = path[depth].p_ext;
3365 ee_block = le32_to_cpu(ex->ee_block);
3366 ee_len = ext4_ext_get_actual_len(ext: ex);
3367 unwritten = ext4_ext_is_unwritten(ext: ex);
3368
3369 if (map->m_lblk + map->m_len < ee_block + ee_len) {
3370 split_flag1 = split_flag & EXT4_EXT_MAY_ZEROOUT;
3371 flags1 = flags | EXT4_GET_BLOCKS_PRE_IO;
3372 if (unwritten)
3373 split_flag1 |= EXT4_EXT_MARK_UNWRIT1 |
3374 EXT4_EXT_MARK_UNWRIT2;
3375 if (split_flag & EXT4_EXT_DATA_VALID2)
3376 split_flag1 |= EXT4_EXT_DATA_VALID1;
3377 path = ext4_split_extent_at(handle, inode, path,
3378 split: map->m_lblk + map->m_len, split_flag: split_flag1, flags: flags1);
3379 if (IS_ERR(ptr: path))
3380 return path;
3381 /*
3382 * Update path is required because previous ext4_split_extent_at
3383 * may result in split of original leaf or extent zeroout.
3384 */
3385 path = ext4_find_extent(inode, block: map->m_lblk, path, flags);
3386 if (IS_ERR(ptr: path))
3387 return path;
3388 depth = ext_depth(inode);
3389 ex = path[depth].p_ext;
3390 if (!ex) {
3391 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
3392 (unsigned long) map->m_lblk);
3393 ext4_free_ext_path(path);
3394 return ERR_PTR(error: -EFSCORRUPTED);
3395 }
3396 unwritten = ext4_ext_is_unwritten(ext: ex);
3397 }
3398
3399 if (map->m_lblk >= ee_block) {
3400 split_flag1 = split_flag & EXT4_EXT_DATA_VALID2;
3401 if (unwritten) {
3402 split_flag1 |= EXT4_EXT_MARK_UNWRIT1;
3403 split_flag1 |= split_flag & (EXT4_EXT_MAY_ZEROOUT |
3404 EXT4_EXT_MARK_UNWRIT2);
3405 }
3406 path = ext4_split_extent_at(handle, inode, path,
3407 split: map->m_lblk, split_flag: split_flag1, flags);
3408 if (IS_ERR(ptr: path))
3409 return path;
3410 }
3411
3412 if (allocated) {
3413 if (map->m_lblk + map->m_len > ee_block + ee_len)
3414 *allocated = ee_len - (map->m_lblk - ee_block);
3415 else
3416 *allocated = map->m_len;
3417 }
3418 ext4_ext_show_leaf(inode, path);
3419 return path;
3420}
3421
3422/*
3423 * This function is called by ext4_ext_map_blocks() if someone tries to write
3424 * to an unwritten extent. It may result in splitting the unwritten
3425 * extent into multiple extents (up to three - one initialized and two
3426 * unwritten).
3427 * There are three possibilities:
3428 * a> There is no split required: Entire extent should be initialized
3429 * b> Splits in two extents: Write is happening at either end of the extent
3430 * c> Splits in three extents: Somone is writing in middle of the extent
3431 *
3432 * Pre-conditions:
3433 * - The extent pointed to by 'path' is unwritten.
3434 * - The extent pointed to by 'path' contains a superset
3435 * of the logical span [map->m_lblk, map->m_lblk + map->m_len).
3436 *
3437 * Post-conditions on success:
3438 * - the returned value is the number of blocks beyond map->l_lblk
3439 * that are allocated and initialized.
3440 * It is guaranteed to be >= map->m_len.
3441 */
3442static struct ext4_ext_path *
3443ext4_ext_convert_to_initialized(handle_t *handle, struct inode *inode,
3444 struct ext4_map_blocks *map, struct ext4_ext_path *path,
3445 int flags, unsigned int *allocated)
3446{
3447 struct ext4_sb_info *sbi;
3448 struct ext4_extent_header *eh;
3449 struct ext4_map_blocks split_map;
3450 struct ext4_extent zero_ex1, zero_ex2;
3451 struct ext4_extent *ex, *abut_ex;
3452 ext4_lblk_t ee_block, eof_block;
3453 unsigned int ee_len, depth, map_len = map->m_len;
3454 int err = 0;
3455 int split_flag = EXT4_EXT_DATA_VALID2;
3456 unsigned int max_zeroout = 0;
3457
3458 ext_debug(inode, "logical block %llu, max_blocks %u\n",
3459 (unsigned long long)map->m_lblk, map_len);
3460
3461 sbi = EXT4_SB(sb: inode->i_sb);
3462 eof_block = (EXT4_I(inode)->i_disksize + inode->i_sb->s_blocksize - 1)
3463 >> inode->i_sb->s_blocksize_bits;
3464 if (eof_block < map->m_lblk + map_len)
3465 eof_block = map->m_lblk + map_len;
3466
3467 depth = ext_depth(inode);
3468 eh = path[depth].p_hdr;
3469 ex = path[depth].p_ext;
3470 ee_block = le32_to_cpu(ex->ee_block);
3471 ee_len = ext4_ext_get_actual_len(ext: ex);
3472 zero_ex1.ee_len = 0;
3473 zero_ex2.ee_len = 0;
3474
3475 trace_ext4_ext_convert_to_initialized_enter(inode, map, ux: ex);
3476
3477 /* Pre-conditions */
3478 BUG_ON(!ext4_ext_is_unwritten(ex));
3479 BUG_ON(!in_range(map->m_lblk, ee_block, ee_len));
3480
3481 /*
3482 * Attempt to transfer newly initialized blocks from the currently
3483 * unwritten extent to its neighbor. This is much cheaper
3484 * than an insertion followed by a merge as those involve costly
3485 * memmove() calls. Transferring to the left is the common case in
3486 * steady state for workloads doing fallocate(FALLOC_FL_KEEP_SIZE)
3487 * followed by append writes.
3488 *
3489 * Limitations of the current logic:
3490 * - L1: we do not deal with writes covering the whole extent.
3491 * This would require removing the extent if the transfer
3492 * is possible.
3493 * - L2: we only attempt to merge with an extent stored in the
3494 * same extent tree node.
3495 */
3496 *allocated = 0;
3497 if ((map->m_lblk == ee_block) &&
3498 /* See if we can merge left */
3499 (map_len < ee_len) && /*L1*/
3500 (ex > EXT_FIRST_EXTENT(eh))) { /*L2*/
3501 ext4_lblk_t prev_lblk;
3502 ext4_fsblk_t prev_pblk, ee_pblk;
3503 unsigned int prev_len;
3504
3505 abut_ex = ex - 1;
3506 prev_lblk = le32_to_cpu(abut_ex->ee_block);
3507 prev_len = ext4_ext_get_actual_len(ext: abut_ex);
3508 prev_pblk = ext4_ext_pblock(ex: abut_ex);
3509 ee_pblk = ext4_ext_pblock(ex);
3510
3511 /*
3512 * A transfer of blocks from 'ex' to 'abut_ex' is allowed
3513 * upon those conditions:
3514 * - C1: abut_ex is initialized,
3515 * - C2: abut_ex is logically abutting ex,
3516 * - C3: abut_ex is physically abutting ex,
3517 * - C4: abut_ex can receive the additional blocks without
3518 * overflowing the (initialized) length limit.
3519 */
3520 if ((!ext4_ext_is_unwritten(ext: abut_ex)) && /*C1*/
3521 ((prev_lblk + prev_len) == ee_block) && /*C2*/
3522 ((prev_pblk + prev_len) == ee_pblk) && /*C3*/
3523 (prev_len < (EXT_INIT_MAX_LEN - map_len))) { /*C4*/
3524 err = ext4_ext_get_access(handle, inode, path: path + depth);
3525 if (err)
3526 goto errout;
3527
3528 trace_ext4_ext_convert_to_initialized_fastpath(inode,
3529 map, ux: ex, ix: abut_ex);
3530
3531 /* Shift the start of ex by 'map_len' blocks */
3532 ex->ee_block = cpu_to_le32(ee_block + map_len);
3533 ext4_ext_store_pblock(ex, pb: ee_pblk + map_len);
3534 ex->ee_len = cpu_to_le16(ee_len - map_len);
3535 ext4_ext_mark_unwritten(ext: ex); /* Restore the flag */
3536
3537 /* Extend abut_ex by 'map_len' blocks */
3538 abut_ex->ee_len = cpu_to_le16(prev_len + map_len);
3539
3540 /* Result: number of initialized blocks past m_lblk */
3541 *allocated = map_len;
3542 }
3543 } else if (((map->m_lblk + map_len) == (ee_block + ee_len)) &&
3544 (map_len < ee_len) && /*L1*/
3545 ex < EXT_LAST_EXTENT(eh)) { /*L2*/
3546 /* See if we can merge right */
3547 ext4_lblk_t next_lblk;
3548 ext4_fsblk_t next_pblk, ee_pblk;
3549 unsigned int next_len;
3550
3551 abut_ex = ex + 1;
3552 next_lblk = le32_to_cpu(abut_ex->ee_block);
3553 next_len = ext4_ext_get_actual_len(ext: abut_ex);
3554 next_pblk = ext4_ext_pblock(ex: abut_ex);
3555 ee_pblk = ext4_ext_pblock(ex);
3556
3557 /*
3558 * A transfer of blocks from 'ex' to 'abut_ex' is allowed
3559 * upon those conditions:
3560 * - C1: abut_ex is initialized,
3561 * - C2: abut_ex is logically abutting ex,
3562 * - C3: abut_ex is physically abutting ex,
3563 * - C4: abut_ex can receive the additional blocks without
3564 * overflowing the (initialized) length limit.
3565 */
3566 if ((!ext4_ext_is_unwritten(ext: abut_ex)) && /*C1*/
3567 ((map->m_lblk + map_len) == next_lblk) && /*C2*/
3568 ((ee_pblk + ee_len) == next_pblk) && /*C3*/
3569 (next_len < (EXT_INIT_MAX_LEN - map_len))) { /*C4*/
3570 err = ext4_ext_get_access(handle, inode, path: path + depth);
3571 if (err)
3572 goto errout;
3573
3574 trace_ext4_ext_convert_to_initialized_fastpath(inode,
3575 map, ux: ex, ix: abut_ex);
3576
3577 /* Shift the start of abut_ex by 'map_len' blocks */
3578 abut_ex->ee_block = cpu_to_le32(next_lblk - map_len);
3579 ext4_ext_store_pblock(ex: abut_ex, pb: next_pblk - map_len);
3580 ex->ee_len = cpu_to_le16(ee_len - map_len);
3581 ext4_ext_mark_unwritten(ext: ex); /* Restore the flag */
3582
3583 /* Extend abut_ex by 'map_len' blocks */
3584 abut_ex->ee_len = cpu_to_le16(next_len + map_len);
3585
3586 /* Result: number of initialized blocks past m_lblk */
3587 *allocated = map_len;
3588 }
3589 }
3590 if (*allocated) {
3591 /* Mark the block containing both extents as dirty */
3592 err = ext4_ext_dirty(handle, inode, path + depth);
3593
3594 /* Update path to point to the right extent */
3595 path[depth].p_ext = abut_ex;
3596 if (err)
3597 goto errout;
3598 goto out;
3599 } else
3600 *allocated = ee_len - (map->m_lblk - ee_block);
3601
3602 WARN_ON(map->m_lblk < ee_block);
3603 /*
3604 * It is safe to convert extent to initialized via explicit
3605 * zeroout only if extent is fully inside i_size or new_size.
3606 */
3607 split_flag |= ee_block + ee_len <= eof_block ? EXT4_EXT_MAY_ZEROOUT : 0;
3608
3609 if (EXT4_EXT_MAY_ZEROOUT & split_flag)
3610 max_zeroout = sbi->s_extent_max_zeroout_kb >>
3611 (inode->i_sb->s_blocksize_bits - 10);
3612
3613 /*
3614 * five cases:
3615 * 1. split the extent into three extents.
3616 * 2. split the extent into two extents, zeroout the head of the first
3617 * extent.
3618 * 3. split the extent into two extents, zeroout the tail of the second
3619 * extent.
3620 * 4. split the extent into two extents with out zeroout.
3621 * 5. no splitting needed, just possibly zeroout the head and / or the
3622 * tail of the extent.
3623 */
3624 split_map.m_lblk = map->m_lblk;
3625 split_map.m_len = map->m_len;
3626
3627 if (max_zeroout && (*allocated > split_map.m_len)) {
3628 if (*allocated <= max_zeroout) {
3629 /* case 3 or 5 */
3630 zero_ex1.ee_block =
3631 cpu_to_le32(split_map.m_lblk +
3632 split_map.m_len);
3633 zero_ex1.ee_len =
3634 cpu_to_le16(*allocated - split_map.m_len);
3635 ext4_ext_store_pblock(ex: &zero_ex1,
3636 pb: ext4_ext_pblock(ex) + split_map.m_lblk +
3637 split_map.m_len - ee_block);
3638 err = ext4_ext_zeroout(inode, ex: &zero_ex1);
3639 if (err)
3640 goto fallback;
3641 split_map.m_len = *allocated;
3642 }
3643 if (split_map.m_lblk - ee_block + split_map.m_len <
3644 max_zeroout) {
3645 /* case 2 or 5 */
3646 if (split_map.m_lblk != ee_block) {
3647 zero_ex2.ee_block = ex->ee_block;
3648 zero_ex2.ee_len = cpu_to_le16(split_map.m_lblk -
3649 ee_block);
3650 ext4_ext_store_pblock(ex: &zero_ex2,
3651 pb: ext4_ext_pblock(ex));
3652 err = ext4_ext_zeroout(inode, ex: &zero_ex2);
3653 if (err)
3654 goto fallback;
3655 }
3656
3657 split_map.m_len += split_map.m_lblk - ee_block;
3658 split_map.m_lblk = ee_block;
3659 *allocated = map->m_len;
3660 }
3661 }
3662
3663fallback:
3664 path = ext4_split_extent(handle, inode, path, map: &split_map, split_flag,
3665 flags, NULL);
3666 if (IS_ERR(ptr: path))
3667 return path;
3668out:
3669 /* If we have gotten a failure, don't zero out status tree */
3670 ext4_zeroout_es(inode, ex: &zero_ex1);
3671 ext4_zeroout_es(inode, ex: &zero_ex2);
3672 return path;
3673
3674errout:
3675 ext4_free_ext_path(path);
3676 return ERR_PTR(error: err);
3677}
3678
3679/*
3680 * This function is called by ext4_ext_map_blocks() from
3681 * ext4_get_blocks_dio_write() when DIO to write
3682 * to an unwritten extent.
3683 *
3684 * Writing to an unwritten extent may result in splitting the unwritten
3685 * extent into multiple initialized/unwritten extents (up to three)
3686 * There are three possibilities:
3687 * a> There is no split required: Entire extent should be unwritten
3688 * b> Splits in two extents: Write is happening at either end of the extent
3689 * c> Splits in three extents: Somone is writing in middle of the extent
3690 *
3691 * This works the same way in the case of initialized -> unwritten conversion.
3692 *
3693 * One of more index blocks maybe needed if the extent tree grow after
3694 * the unwritten extent split. To prevent ENOSPC occur at the IO
3695 * complete, we need to split the unwritten extent before DIO submit
3696 * the IO. The unwritten extent called at this time will be split
3697 * into three unwritten extent(at most). After IO complete, the part
3698 * being filled will be convert to initialized by the end_io callback function
3699 * via ext4_convert_unwritten_extents().
3700 *
3701 * The size of unwritten extent to be written is passed to the caller via the
3702 * allocated pointer. Return an extent path pointer on success, or an error
3703 * pointer on failure.
3704 */
3705static struct ext4_ext_path *ext4_split_convert_extents(handle_t *handle,
3706 struct inode *inode,
3707 struct ext4_map_blocks *map,
3708 struct ext4_ext_path *path,
3709 int flags, unsigned int *allocated)
3710{
3711 ext4_lblk_t eof_block;
3712 ext4_lblk_t ee_block;
3713 struct ext4_extent *ex;
3714 unsigned int ee_len;
3715 int split_flag = 0, depth;
3716
3717 ext_debug(inode, "logical block %llu, max_blocks %u\n",
3718 (unsigned long long)map->m_lblk, map->m_len);
3719
3720 eof_block = (EXT4_I(inode)->i_disksize + inode->i_sb->s_blocksize - 1)
3721 >> inode->i_sb->s_blocksize_bits;
3722 if (eof_block < map->m_lblk + map->m_len)
3723 eof_block = map->m_lblk + map->m_len;
3724 /*
3725 * It is safe to convert extent to initialized via explicit
3726 * zeroout only if extent is fully inside i_size or new_size.
3727 */
3728 depth = ext_depth(inode);
3729 ex = path[depth].p_ext;
3730 ee_block = le32_to_cpu(ex->ee_block);
3731 ee_len = ext4_ext_get_actual_len(ext: ex);
3732
3733 /* Convert to unwritten */
3734 if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) {
3735 split_flag |= EXT4_EXT_DATA_VALID1;
3736 /* Convert to initialized */
3737 } else if (flags & EXT4_GET_BLOCKS_CONVERT) {
3738 split_flag |= ee_block + ee_len <= eof_block ?
3739 EXT4_EXT_MAY_ZEROOUT : 0;
3740 split_flag |= (EXT4_EXT_MARK_UNWRIT2 | EXT4_EXT_DATA_VALID2);
3741 }
3742 flags |= EXT4_GET_BLOCKS_PRE_IO;
3743 return ext4_split_extent(handle, inode, path, map, split_flag, flags,
3744 allocated);
3745}
3746
3747static struct ext4_ext_path *
3748ext4_convert_unwritten_extents_endio(handle_t *handle, struct inode *inode,
3749 struct ext4_map_blocks *map,
3750 struct ext4_ext_path *path)
3751{
3752 struct ext4_extent *ex;
3753 ext4_lblk_t ee_block;
3754 unsigned int ee_len;
3755 int depth;
3756 int err = 0;
3757
3758 depth = ext_depth(inode);
3759 ex = path[depth].p_ext;
3760 ee_block = le32_to_cpu(ex->ee_block);
3761 ee_len = ext4_ext_get_actual_len(ext: ex);
3762
3763 ext_debug(inode, "logical block %llu, max_blocks %u\n",
3764 (unsigned long long)ee_block, ee_len);
3765
3766 /* If extent is larger than requested it is a clear sign that we still
3767 * have some extent state machine issues left. So extent_split is still
3768 * required.
3769 * TODO: Once all related issues will be fixed this situation should be
3770 * illegal.
3771 */
3772 if (ee_block != map->m_lblk || ee_len > map->m_len) {
3773#ifdef CONFIG_EXT4_DEBUG
3774 ext4_warning(inode->i_sb, "Inode (%ld) finished: extent logical block %llu,"
3775 " len %u; IO logical block %llu, len %u",
3776 inode->i_ino, (unsigned long long)ee_block, ee_len,
3777 (unsigned long long)map->m_lblk, map->m_len);
3778#endif
3779 path = ext4_split_convert_extents(handle, inode, map, path,
3780 EXT4_GET_BLOCKS_CONVERT, NULL);
3781 if (IS_ERR(ptr: path))
3782 return path;
3783
3784 path = ext4_find_extent(inode, block: map->m_lblk, path, flags: 0);
3785 if (IS_ERR(ptr: path))
3786 return path;
3787 depth = ext_depth(inode);
3788 ex = path[depth].p_ext;
3789 }
3790
3791 err = ext4_ext_get_access(handle, inode, path: path + depth);
3792 if (err)
3793 goto errout;
3794 /* first mark the extent as initialized */
3795 ext4_ext_mark_initialized(ext: ex);
3796
3797 /* note: ext4_ext_correct_indexes() isn't needed here because
3798 * borders are not changed
3799 */
3800 ext4_ext_try_to_merge(handle, inode, path, ex);
3801
3802 /* Mark modified extent as dirty */
3803 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3804 if (err)
3805 goto errout;
3806
3807 ext4_ext_show_leaf(inode, path);
3808 return path;
3809
3810errout:
3811 ext4_free_ext_path(path);
3812 return ERR_PTR(error: err);
3813}
3814
3815static struct ext4_ext_path *
3816convert_initialized_extent(handle_t *handle, struct inode *inode,
3817 struct ext4_map_blocks *map,
3818 struct ext4_ext_path *path,
3819 unsigned int *allocated)
3820{
3821 struct ext4_extent *ex;
3822 ext4_lblk_t ee_block;
3823 unsigned int ee_len;
3824 int depth;
3825 int err = 0;
3826
3827 /*
3828 * Make sure that the extent is no bigger than we support with
3829 * unwritten extent
3830 */
3831 if (map->m_len > EXT_UNWRITTEN_MAX_LEN)
3832 map->m_len = EXT_UNWRITTEN_MAX_LEN / 2;
3833
3834 depth = ext_depth(inode);
3835 ex = path[depth].p_ext;
3836 ee_block = le32_to_cpu(ex->ee_block);
3837 ee_len = ext4_ext_get_actual_len(ext: ex);
3838
3839 ext_debug(inode, "logical block %llu, max_blocks %u\n",
3840 (unsigned long long)ee_block, ee_len);
3841
3842 if (ee_block != map->m_lblk || ee_len > map->m_len) {
3843 path = ext4_split_convert_extents(handle, inode, map, path,
3844 EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, NULL);
3845 if (IS_ERR(ptr: path))
3846 return path;
3847
3848 path = ext4_find_extent(inode, block: map->m_lblk, path, flags: 0);
3849 if (IS_ERR(ptr: path))
3850 return path;
3851 depth = ext_depth(inode);
3852 ex = path[depth].p_ext;
3853 if (!ex) {
3854 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
3855 (unsigned long) map->m_lblk);
3856 err = -EFSCORRUPTED;
3857 goto errout;
3858 }
3859 }
3860
3861 err = ext4_ext_get_access(handle, inode, path: path + depth);
3862 if (err)
3863 goto errout;
3864 /* first mark the extent as unwritten */
3865 ext4_ext_mark_unwritten(ext: ex);
3866
3867 /* note: ext4_ext_correct_indexes() isn't needed here because
3868 * borders are not changed
3869 */
3870 ext4_ext_try_to_merge(handle, inode, path, ex);
3871
3872 /* Mark modified extent as dirty */
3873 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3874 if (err)
3875 goto errout;
3876 ext4_ext_show_leaf(inode, path);
3877
3878 ext4_update_inode_fsync_trans(handle, inode, datasync: 1);
3879
3880 map->m_flags |= EXT4_MAP_UNWRITTEN;
3881 if (*allocated > map->m_len)
3882 *allocated = map->m_len;
3883 map->m_len = *allocated;
3884 return path;
3885
3886errout:
3887 ext4_free_ext_path(path);
3888 return ERR_PTR(error: err);
3889}
3890
3891static struct ext4_ext_path *
3892ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode,
3893 struct ext4_map_blocks *map,
3894 struct ext4_ext_path *path, int flags,
3895 unsigned int *allocated, ext4_fsblk_t newblock)
3896{
3897 int err = 0;
3898
3899 ext_debug(inode, "logical block %llu, max_blocks %u, flags 0x%x, allocated %u\n",
3900 (unsigned long long)map->m_lblk, map->m_len, flags,
3901 *allocated);
3902 ext4_ext_show_leaf(inode, path);
3903
3904 /*
3905 * When writing into unwritten space, we should not fail to
3906 * allocate metadata blocks for the new extent block if needed.
3907 */
3908 flags |= EXT4_GET_BLOCKS_METADATA_NOFAIL;
3909
3910 trace_ext4_ext_handle_unwritten_extents(inode, map, flags,
3911 allocated: *allocated, newblock);
3912
3913 /* get_block() before submitting IO, split the extent */
3914 if (flags & EXT4_GET_BLOCKS_PRE_IO) {
3915 path = ext4_split_convert_extents(handle, inode, map, path,
3916 flags: flags | EXT4_GET_BLOCKS_CONVERT, allocated);
3917 if (IS_ERR(ptr: path))
3918 return path;
3919 /*
3920 * shouldn't get a 0 allocated when splitting an extent unless
3921 * m_len is 0 (bug) or extent has been corrupted
3922 */
3923 if (unlikely(*allocated == 0)) {
3924 EXT4_ERROR_INODE(inode,
3925 "unexpected allocated == 0, m_len = %u",
3926 map->m_len);
3927 err = -EFSCORRUPTED;
3928 goto errout;
3929 }
3930 map->m_flags |= EXT4_MAP_UNWRITTEN;
3931 goto out;
3932 }
3933 /* IO end_io complete, convert the filled extent to written */
3934 if (flags & EXT4_GET_BLOCKS_CONVERT) {
3935 path = ext4_convert_unwritten_extents_endio(handle, inode,
3936 map, path);
3937 if (IS_ERR(ptr: path))
3938 return path;
3939 ext4_update_inode_fsync_trans(handle, inode, datasync: 1);
3940 goto map_out;
3941 }
3942 /* buffered IO cases */
3943 /*
3944 * repeat fallocate creation request
3945 * we already have an unwritten extent
3946 */
3947 if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) {
3948 map->m_flags |= EXT4_MAP_UNWRITTEN;
3949 goto map_out;
3950 }
3951
3952 /* buffered READ or buffered write_begin() lookup */
3953 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) {
3954 /*
3955 * We have blocks reserved already. We
3956 * return allocated blocks so that delalloc
3957 * won't do block reservation for us. But
3958 * the buffer head will be unmapped so that
3959 * a read from the block returns 0s.
3960 */
3961 map->m_flags |= EXT4_MAP_UNWRITTEN;
3962 goto out1;
3963 }
3964
3965 /*
3966 * Default case when (flags & EXT4_GET_BLOCKS_CREATE) == 1.
3967 * For buffered writes, at writepage time, etc. Convert a
3968 * discovered unwritten extent to written.
3969 */
3970 path = ext4_ext_convert_to_initialized(handle, inode, map, path,
3971 flags, allocated);
3972 if (IS_ERR(ptr: path))
3973 return path;
3974 ext4_update_inode_fsync_trans(handle, inode, datasync: 1);
3975 /*
3976 * shouldn't get a 0 allocated when converting an unwritten extent
3977 * unless m_len is 0 (bug) or extent has been corrupted
3978 */
3979 if (unlikely(*allocated == 0)) {
3980 EXT4_ERROR_INODE(inode, "unexpected allocated == 0, m_len = %u",
3981 map->m_len);
3982 err = -EFSCORRUPTED;
3983 goto errout;
3984 }
3985
3986out:
3987 map->m_flags |= EXT4_MAP_NEW;
3988map_out:
3989 map->m_flags |= EXT4_MAP_MAPPED;
3990out1:
3991 map->m_pblk = newblock;
3992 if (*allocated > map->m_len)
3993 *allocated = map->m_len;
3994 map->m_len = *allocated;
3995 ext4_ext_show_leaf(inode, path);
3996 return path;
3997
3998errout:
3999 ext4_free_ext_path(path);
4000 return ERR_PTR(error: err);
4001}
4002
4003/*
4004 * get_implied_cluster_alloc - check to see if the requested
4005 * allocation (in the map structure) overlaps with a cluster already
4006 * allocated in an extent.
4007 * @sb The filesystem superblock structure
4008 * @map The requested lblk->pblk mapping
4009 * @ex The extent structure which might contain an implied
4010 * cluster allocation
4011 *
4012 * This function is called by ext4_ext_map_blocks() after we failed to
4013 * find blocks that were already in the inode's extent tree. Hence,
4014 * we know that the beginning of the requested region cannot overlap
4015 * the extent from the inode's extent tree. There are three cases we
4016 * want to catch. The first is this case:
4017 *
4018 * |--- cluster # N--|
4019 * |--- extent ---| |---- requested region ---|
4020 * |==========|
4021 *
4022 * The second case that we need to test for is this one:
4023 *
4024 * |--------- cluster # N ----------------|
4025 * |--- requested region --| |------- extent ----|
4026 * |=======================|
4027 *
4028 * The third case is when the requested region lies between two extents
4029 * within the same cluster:
4030 * |------------- cluster # N-------------|
4031 * |----- ex -----| |---- ex_right ----|
4032 * |------ requested region ------|
4033 * |================|
4034 *
4035 * In each of the above cases, we need to set the map->m_pblk and
4036 * map->m_len so it corresponds to the return the extent labelled as
4037 * "|====|" from cluster #N, since it is already in use for data in
4038 * cluster EXT4_B2C(sbi, map->m_lblk). We will then return 1 to
4039 * signal to ext4_ext_map_blocks() that map->m_pblk should be treated
4040 * as a new "allocated" block region. Otherwise, we will return 0 and
4041 * ext4_ext_map_blocks() will then allocate one or more new clusters
4042 * by calling ext4_mb_new_blocks().
4043 */
4044static int get_implied_cluster_alloc(struct super_block *sb,
4045 struct ext4_map_blocks *map,
4046 struct ext4_extent *ex,
4047 struct ext4_ext_path *path)
4048{
4049 struct ext4_sb_info *sbi = EXT4_SB(sb);
4050 ext4_lblk_t c_offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
4051 ext4_lblk_t ex_cluster_start, ex_cluster_end;
4052 ext4_lblk_t rr_cluster_start;
4053 ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block);
4054 ext4_fsblk_t ee_start = ext4_ext_pblock(ex);
4055 unsigned short ee_len = ext4_ext_get_actual_len(ext: ex);
4056
4057 /* The extent passed in that we are trying to match */
4058 ex_cluster_start = EXT4_B2C(sbi, ee_block);
4059 ex_cluster_end = EXT4_B2C(sbi, ee_block + ee_len - 1);
4060
4061 /* The requested region passed into ext4_map_blocks() */
4062 rr_cluster_start = EXT4_B2C(sbi, map->m_lblk);
4063
4064 if ((rr_cluster_start == ex_cluster_end) ||
4065 (rr_cluster_start == ex_cluster_start)) {
4066 if (rr_cluster_start == ex_cluster_end)
4067 ee_start += ee_len - 1;
4068 map->m_pblk = EXT4_PBLK_CMASK(sbi, ee_start) + c_offset;
4069 map->m_len = min(map->m_len,
4070 (unsigned) sbi->s_cluster_ratio - c_offset);
4071 /*
4072 * Check for and handle this case:
4073 *
4074 * |--------- cluster # N-------------|
4075 * |------- extent ----|
4076 * |--- requested region ---|
4077 * |===========|
4078 */
4079
4080 if (map->m_lblk < ee_block)
4081 map->m_len = min(map->m_len, ee_block - map->m_lblk);
4082
4083 /*
4084 * Check for the case where there is already another allocated
4085 * block to the right of 'ex' but before the end of the cluster.
4086 *
4087 * |------------- cluster # N-------------|
4088 * |----- ex -----| |---- ex_right ----|
4089 * |------ requested region ------|
4090 * |================|
4091 */
4092 if (map->m_lblk > ee_block) {
4093 ext4_lblk_t next = ext4_ext_next_allocated_block(path);
4094 map->m_len = min(map->m_len, next - map->m_lblk);
4095 }
4096
4097 trace_ext4_get_implied_cluster_alloc_exit(sb, map, ret: 1);
4098 return 1;
4099 }
4100
4101 trace_ext4_get_implied_cluster_alloc_exit(sb, map, ret: 0);
4102 return 0;
4103}
4104
4105/*
4106 * Determine hole length around the given logical block, first try to
4107 * locate and expand the hole from the given @path, and then adjust it
4108 * if it's partially or completely converted to delayed extents, insert
4109 * it into the extent cache tree if it's indeed a hole, finally return
4110 * the length of the determined extent.
4111 */
4112static ext4_lblk_t ext4_ext_determine_insert_hole(struct inode *inode,
4113 struct ext4_ext_path *path,
4114 ext4_lblk_t lblk)
4115{
4116 ext4_lblk_t hole_start, len;
4117 struct extent_status es;
4118
4119 hole_start = lblk;
4120 len = ext4_ext_find_hole(inode, path, lblk: &hole_start);
4121again:
4122 ext4_es_find_extent_range(inode, match_fn: &ext4_es_is_delayed, lblk: hole_start,
4123 end: hole_start + len - 1, es: &es);
4124 if (!es.es_len)
4125 goto insert_hole;
4126
4127 /*
4128 * There's a delalloc extent in the hole, handle it if the delalloc
4129 * extent is in front of, behind and straddle the queried range.
4130 */
4131 if (lblk >= es.es_lblk + es.es_len) {
4132 /*
4133 * The delalloc extent is in front of the queried range,
4134 * find again from the queried start block.
4135 */
4136 len -= lblk - hole_start;
4137 hole_start = lblk;
4138 goto again;
4139 } else if (in_range(lblk, es.es_lblk, es.es_len)) {
4140 /*
4141 * The delalloc extent containing lblk, it must have been
4142 * added after ext4_map_blocks() checked the extent status
4143 * tree so we are not holding i_rwsem and delalloc info is
4144 * only stabilized by i_data_sem we are going to release
4145 * soon. Don't modify the extent status tree and report
4146 * extent as a hole, just adjust the length to the delalloc
4147 * extent's after lblk.
4148 */
4149 len = es.es_lblk + es.es_len - lblk;
4150 return len;
4151 } else {
4152 /*
4153 * The delalloc extent is partially or completely behind
4154 * the queried range, update hole length until the
4155 * beginning of the delalloc extent.
4156 */
4157 len = min(es.es_lblk - hole_start, len);
4158 }
4159
4160insert_hole:
4161 /* Put just found gap into cache to speed up subsequent requests */
4162 ext_debug(inode, " -> %u:%u\n", hole_start, len);
4163 ext4_es_insert_extent(inode, lblk: hole_start, len, pblk: ~0,
4164 EXTENT_STATUS_HOLE, delalloc_reserve_used: false);
4165
4166 /* Update hole_len to reflect hole size after lblk */
4167 if (hole_start != lblk)
4168 len -= lblk - hole_start;
4169
4170 return len;
4171}
4172
4173/*
4174 * Block allocation/map/preallocation routine for extents based files
4175 *
4176 *
4177 * Need to be called with
4178 * down_read(&EXT4_I(inode)->i_data_sem) if not allocating file system block
4179 * (ie, flags is zero). Otherwise down_write(&EXT4_I(inode)->i_data_sem)
4180 *
4181 * return > 0, number of blocks already mapped/allocated
4182 * if flags doesn't contain EXT4_GET_BLOCKS_CREATE and these are pre-allocated blocks
4183 * buffer head is unmapped
4184 * otherwise blocks are mapped
4185 *
4186 * return = 0, if plain look up failed (blocks have not been allocated)
4187 * buffer head is unmapped
4188 *
4189 * return < 0, error case.
4190 */
4191int ext4_ext_map_blocks(handle_t *handle, struct inode *inode,
4192 struct ext4_map_blocks *map, int flags)
4193{
4194 struct ext4_ext_path *path = NULL;
4195 struct ext4_extent newex, *ex, ex2;
4196 struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
4197 ext4_fsblk_t newblock = 0, pblk;
4198 int err = 0, depth;
4199 unsigned int allocated = 0, offset = 0;
4200 unsigned int allocated_clusters = 0;
4201 struct ext4_allocation_request ar;
4202 ext4_lblk_t cluster_offset;
4203
4204 ext_debug(inode, "blocks %u/%u requested\n", map->m_lblk, map->m_len);
4205 trace_ext4_ext_map_blocks_enter(inode, lblk: map->m_lblk, len: map->m_len, flags);
4206
4207 /* find extent for this block */
4208 path = ext4_find_extent(inode, block: map->m_lblk, NULL, flags);
4209 if (IS_ERR(ptr: path)) {
4210 err = PTR_ERR(ptr: path);
4211 goto out;
4212 }
4213
4214 depth = ext_depth(inode);
4215
4216 /*
4217 * consistent leaf must not be empty;
4218 * this situation is possible, though, _during_ tree modification;
4219 * this is why assert can't be put in ext4_find_extent()
4220 */
4221 if (unlikely(path[depth].p_ext == NULL && depth != 0)) {
4222 EXT4_ERROR_INODE(inode, "bad extent address "
4223 "lblock: %lu, depth: %d pblock %lld",
4224 (unsigned long) map->m_lblk, depth,
4225 path[depth].p_block);
4226 err = -EFSCORRUPTED;
4227 goto out;
4228 }
4229
4230 ex = path[depth].p_ext;
4231 if (ex) {
4232 ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block);
4233 ext4_fsblk_t ee_start = ext4_ext_pblock(ex);
4234 unsigned short ee_len;
4235
4236
4237 /*
4238 * unwritten extents are treated as holes, except that
4239 * we split out initialized portions during a write.
4240 */
4241 ee_len = ext4_ext_get_actual_len(ext: ex);
4242
4243 trace_ext4_ext_show_extent(inode, lblk: ee_block, pblk: ee_start, len: ee_len);
4244
4245 /* if found extent covers block, simply return it */
4246 if (in_range(map->m_lblk, ee_block, ee_len)) {
4247 newblock = map->m_lblk - ee_block + ee_start;
4248 /* number of remaining blocks in the extent */
4249 allocated = ee_len - (map->m_lblk - ee_block);
4250 ext_debug(inode, "%u fit into %u:%d -> %llu\n",
4251 map->m_lblk, ee_block, ee_len, newblock);
4252
4253 /*
4254 * If the extent is initialized check whether the
4255 * caller wants to convert it to unwritten.
4256 */
4257 if ((!ext4_ext_is_unwritten(ext: ex)) &&
4258 (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) {
4259 path = convert_initialized_extent(handle,
4260 inode, map, path, allocated: &allocated);
4261 if (IS_ERR(ptr: path))
4262 err = PTR_ERR(ptr: path);
4263 goto out;
4264 } else if (!ext4_ext_is_unwritten(ext: ex)) {
4265 map->m_flags |= EXT4_MAP_MAPPED;
4266 map->m_pblk = newblock;
4267 if (allocated > map->m_len)
4268 allocated = map->m_len;
4269 map->m_len = allocated;
4270 ext4_ext_show_leaf(inode, path);
4271 goto out;
4272 }
4273
4274 path = ext4_ext_handle_unwritten_extents(
4275 handle, inode, map, path, flags,
4276 allocated: &allocated, newblock);
4277 if (IS_ERR(ptr: path))
4278 err = PTR_ERR(ptr: path);
4279 goto out;
4280 }
4281 }
4282
4283 /*
4284 * requested block isn't allocated yet;
4285 * we couldn't try to create block if flags doesn't contain EXT4_GET_BLOCKS_CREATE
4286 */
4287 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) {
4288 ext4_lblk_t len;
4289
4290 len = ext4_ext_determine_insert_hole(inode, path, lblk: map->m_lblk);
4291
4292 map->m_pblk = 0;
4293 map->m_len = min_t(unsigned int, map->m_len, len);
4294 goto out;
4295 }
4296
4297 /*
4298 * Okay, we need to do block allocation.
4299 */
4300 newex.ee_block = cpu_to_le32(map->m_lblk);
4301 cluster_offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
4302
4303 /*
4304 * If we are doing bigalloc, check to see if the extent returned
4305 * by ext4_find_extent() implies a cluster we can use.
4306 */
4307 if (cluster_offset && ex &&
4308 get_implied_cluster_alloc(sb: inode->i_sb, map, ex, path)) {
4309 ar.len = allocated = map->m_len;
4310 newblock = map->m_pblk;
4311 goto got_allocated_blocks;
4312 }
4313
4314 /* find neighbour allocated blocks */
4315 ar.lleft = map->m_lblk;
4316 err = ext4_ext_search_left(inode, path, logical: &ar.lleft, phys: &ar.pleft);
4317 if (err)
4318 goto out;
4319 ar.lright = map->m_lblk;
4320 err = ext4_ext_search_right(inode, path, logical: &ar.lright, phys: &ar.pright,
4321 ret_ex: &ex2, flags);
4322 if (err < 0)
4323 goto out;
4324
4325 /* Check if the extent after searching to the right implies a
4326 * cluster we can use. */
4327 if ((sbi->s_cluster_ratio > 1) && err &&
4328 get_implied_cluster_alloc(sb: inode->i_sb, map, ex: &ex2, path)) {
4329 ar.len = allocated = map->m_len;
4330 newblock = map->m_pblk;
4331 err = 0;
4332 goto got_allocated_blocks;
4333 }
4334
4335 /*
4336 * See if request is beyond maximum number of blocks we can have in
4337 * a single extent. For an initialized extent this limit is
4338 * EXT_INIT_MAX_LEN and for an unwritten extent this limit is
4339 * EXT_UNWRITTEN_MAX_LEN.
4340 */
4341 if (map->m_len > EXT_INIT_MAX_LEN &&
4342 !(flags & EXT4_GET_BLOCKS_UNWRIT_EXT))
4343 map->m_len = EXT_INIT_MAX_LEN;
4344 else if (map->m_len > EXT_UNWRITTEN_MAX_LEN &&
4345 (flags & EXT4_GET_BLOCKS_UNWRIT_EXT))
4346 map->m_len = EXT_UNWRITTEN_MAX_LEN;
4347
4348 /* Check if we can really insert (m_lblk)::(m_lblk + m_len) extent */
4349 newex.ee_len = cpu_to_le16(map->m_len);
4350 err = ext4_ext_check_overlap(sbi, inode, newext: &newex, path);
4351 if (err)
4352 allocated = ext4_ext_get_actual_len(ext: &newex);
4353 else
4354 allocated = map->m_len;
4355
4356 /* allocate new block */
4357 ar.inode = inode;
4358 ar.goal = ext4_ext_find_goal(inode, path, block: map->m_lblk);
4359 ar.logical = map->m_lblk;
4360 /*
4361 * We calculate the offset from the beginning of the cluster
4362 * for the logical block number, since when we allocate a
4363 * physical cluster, the physical block should start at the
4364 * same offset from the beginning of the cluster. This is
4365 * needed so that future calls to get_implied_cluster_alloc()
4366 * work correctly.
4367 */
4368 offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
4369 ar.len = EXT4_NUM_B2C(sbi, offset+allocated);
4370 ar.goal -= offset;
4371 ar.logical -= offset;
4372 if (S_ISREG(inode->i_mode))
4373 ar.flags = EXT4_MB_HINT_DATA;
4374 else
4375 /* disable in-core preallocation for non-regular files */
4376 ar.flags = 0;
4377 if (flags & EXT4_GET_BLOCKS_NO_NORMALIZE)
4378 ar.flags |= EXT4_MB_HINT_NOPREALLOC;
4379 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
4380 ar.flags |= EXT4_MB_DELALLOC_RESERVED;
4381 if (flags & EXT4_GET_BLOCKS_METADATA_NOFAIL)
4382 ar.flags |= EXT4_MB_USE_RESERVED;
4383 newblock = ext4_mb_new_blocks(handle, &ar, &err);
4384 if (!newblock)
4385 goto out;
4386 allocated_clusters = ar.len;
4387 ar.len = EXT4_C2B(sbi, ar.len) - offset;
4388 ext_debug(inode, "allocate new block: goal %llu, found %llu/%u, requested %u\n",
4389 ar.goal, newblock, ar.len, allocated);
4390 if (ar.len > allocated)
4391 ar.len = allocated;
4392
4393got_allocated_blocks:
4394 /* try to insert new extent into found leaf and return */
4395 pblk = newblock + offset;
4396 ext4_ext_store_pblock(ex: &newex, pb: pblk);
4397 newex.ee_len = cpu_to_le16(ar.len);
4398 /* Mark unwritten */
4399 if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) {
4400 ext4_ext_mark_unwritten(ext: &newex);
4401 map->m_flags |= EXT4_MAP_UNWRITTEN;
4402 }
4403
4404 path = ext4_ext_insert_extent(handle, inode, path, newext: &newex, gb_flags: flags);
4405 if (IS_ERR(ptr: path)) {
4406 err = PTR_ERR(ptr: path);
4407 if (allocated_clusters) {
4408 int fb_flags = 0;
4409
4410 /*
4411 * free data blocks we just allocated.
4412 * not a good idea to call discard here directly,
4413 * but otherwise we'd need to call it every free().
4414 */
4415 ext4_discard_preallocations(inode);
4416 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
4417 fb_flags = EXT4_FREE_BLOCKS_NO_QUOT_UPDATE;
4418 ext4_free_blocks(handle, inode, NULL, block: newblock,
4419 EXT4_C2B(sbi, allocated_clusters),
4420 flags: fb_flags);
4421 }
4422 goto out;
4423 }
4424
4425 /*
4426 * Cache the extent and update transaction to commit on fdatasync only
4427 * when it is _not_ an unwritten extent.
4428 */
4429 if ((flags & EXT4_GET_BLOCKS_UNWRIT_EXT) == 0)
4430 ext4_update_inode_fsync_trans(handle, inode, datasync: 1);
4431 else
4432 ext4_update_inode_fsync_trans(handle, inode, datasync: 0);
4433
4434 map->m_flags |= (EXT4_MAP_NEW | EXT4_MAP_MAPPED);
4435 map->m_pblk = pblk;
4436 map->m_len = ar.len;
4437 allocated = map->m_len;
4438 ext4_ext_show_leaf(inode, path);
4439out:
4440 /*
4441 * We never use EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF with CREATE flag.
4442 * So we know that the depth used here is correct, since there was no
4443 * block allocation done if EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF is set.
4444 * If tomorrow we start using this QUERY flag with CREATE, then we will
4445 * need to re-calculate the depth as it might have changed due to block
4446 * allocation.
4447 */
4448 if (flags & EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF) {
4449 WARN_ON_ONCE(flags & EXT4_GET_BLOCKS_CREATE);
4450 if (!err && ex && (ex == EXT_LAST_EXTENT(path[depth].p_hdr)))
4451 map->m_flags |= EXT4_MAP_QUERY_LAST_IN_LEAF;
4452 }
4453
4454 ext4_free_ext_path(path);
4455
4456 trace_ext4_ext_map_blocks_exit(inode, flags, map,
4457 ret: err ? err : allocated);
4458 return err ? err : allocated;
4459}
4460
4461int ext4_ext_truncate(handle_t *handle, struct inode *inode)
4462{
4463 struct super_block *sb = inode->i_sb;
4464 ext4_lblk_t last_block;
4465 int err = 0;
4466
4467 /*
4468 * TODO: optimization is possible here.
4469 * Probably we need not scan at all,
4470 * because page truncation is enough.
4471 */
4472
4473 /* we have to know where to truncate from in crash case */
4474 EXT4_I(inode)->i_disksize = inode->i_size;
4475 err = ext4_mark_inode_dirty(handle, inode);
4476 if (err)
4477 return err;
4478
4479 last_block = (inode->i_size + sb->s_blocksize - 1)
4480 >> EXT4_BLOCK_SIZE_BITS(sb);
4481 ext4_es_remove_extent(inode, lblk: last_block, EXT_MAX_BLOCKS - last_block);
4482
4483retry_remove_space:
4484 err = ext4_ext_remove_space(inode, start: last_block, EXT_MAX_BLOCKS - 1);
4485 if (err == -ENOMEM) {
4486 memalloc_retry_wait(GFP_ATOMIC);
4487 goto retry_remove_space;
4488 }
4489 return err;
4490}
4491
4492static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset,
4493 ext4_lblk_t len, loff_t new_size,
4494 int flags)
4495{
4496 struct inode *inode = file_inode(f: file);
4497 handle_t *handle;
4498 int ret = 0, ret2 = 0, ret3 = 0;
4499 int retries = 0;
4500 int depth = 0;
4501 struct ext4_map_blocks map;
4502 unsigned int credits;
4503 loff_t epos, old_size = i_size_read(inode);
4504 unsigned int blkbits = inode->i_blkbits;
4505 bool alloc_zero = false;
4506
4507 BUG_ON(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS));
4508 map.m_lblk = offset;
4509 map.m_len = len;
4510 /*
4511 * Don't normalize the request if it can fit in one extent so
4512 * that it doesn't get unnecessarily split into multiple
4513 * extents.
4514 */
4515 if (len <= EXT_UNWRITTEN_MAX_LEN)
4516 flags |= EXT4_GET_BLOCKS_NO_NORMALIZE;
4517
4518 /*
4519 * Do the actual write zero during a running journal transaction
4520 * costs a lot. First allocate an unwritten extent and then
4521 * convert it to written after zeroing it out.
4522 */
4523 if (flags & EXT4_GET_BLOCKS_ZERO) {
4524 flags &= ~EXT4_GET_BLOCKS_ZERO;
4525 flags |= EXT4_GET_BLOCKS_UNWRIT_EXT;
4526 alloc_zero = true;
4527 }
4528
4529 /*
4530 * credits to insert 1 extent into extent tree
4531 */
4532 credits = ext4_chunk_trans_blocks(inode, nrblocks: len);
4533 depth = ext_depth(inode);
4534
4535retry:
4536 while (len) {
4537 /*
4538 * Recalculate credits when extent tree depth changes.
4539 */
4540 if (depth != ext_depth(inode)) {
4541 credits = ext4_chunk_trans_blocks(inode, nrblocks: len);
4542 depth = ext_depth(inode);
4543 }
4544
4545 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
4546 credits);
4547 if (IS_ERR(ptr: handle)) {
4548 ret = PTR_ERR(ptr: handle);
4549 break;
4550 }
4551 ret = ext4_map_blocks(handle, inode, map: &map, flags);
4552 if (ret <= 0) {
4553 ext4_debug("inode #%lu: block %u: len %u: "
4554 "ext4_ext_map_blocks returned %d",
4555 inode->i_ino, map.m_lblk,
4556 map.m_len, ret);
4557 ext4_mark_inode_dirty(handle, inode);
4558 ext4_journal_stop(handle);
4559 break;
4560 }
4561 /*
4562 * allow a full retry cycle for any remaining allocations
4563 */
4564 retries = 0;
4565 epos = (loff_t)(map.m_lblk + ret) << blkbits;
4566 inode_set_ctime_current(inode);
4567 if (new_size) {
4568 if (epos > new_size)
4569 epos = new_size;
4570 if (ext4_update_inode_size(inode, newsize: epos) & 0x1)
4571 inode_set_mtime_to_ts(inode,
4572 ts: inode_get_ctime(inode));
4573 if (epos > old_size) {
4574 pagecache_isize_extended(inode, from: old_size, to: epos);
4575 ext4_zero_partial_blocks(handle, inode,
4576 lstart: old_size, lend: epos - old_size);
4577 }
4578 }
4579 ret2 = ext4_mark_inode_dirty(handle, inode);
4580 ext4_update_inode_fsync_trans(handle, inode, datasync: 1);
4581 ret3 = ext4_journal_stop(handle);
4582 ret2 = ret3 ? ret3 : ret2;
4583 if (unlikely(ret2))
4584 break;
4585
4586 if (alloc_zero &&
4587 (map.m_flags & (EXT4_MAP_MAPPED | EXT4_MAP_UNWRITTEN))) {
4588 ret2 = ext4_issue_zeroout(inode, lblk: map.m_lblk, pblk: map.m_pblk,
4589 len: map.m_len);
4590 if (likely(!ret2))
4591 ret2 = ext4_convert_unwritten_extents(NULL,
4592 inode, offset: (loff_t)map.m_lblk << blkbits,
4593 len: (loff_t)map.m_len << blkbits);
4594 if (ret2)
4595 break;
4596 }
4597
4598 map.m_lblk += ret;
4599 map.m_len = len = len - ret;
4600 }
4601 if (ret == -ENOSPC && ext4_should_retry_alloc(sb: inode->i_sb, retries: &retries))
4602 goto retry;
4603
4604 return ret > 0 ? ret2 : ret;
4605}
4606
4607static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len);
4608
4609static int ext4_insert_range(struct file *file, loff_t offset, loff_t len);
4610
4611static long ext4_zero_range(struct file *file, loff_t offset,
4612 loff_t len, int mode)
4613{
4614 struct inode *inode = file_inode(f: file);
4615 handle_t *handle = NULL;
4616 loff_t new_size = 0;
4617 loff_t end = offset + len;
4618 ext4_lblk_t start_lblk, end_lblk;
4619 unsigned int blocksize = i_blocksize(node: inode);
4620 unsigned int blkbits = inode->i_blkbits;
4621 int ret, flags, credits;
4622
4623 trace_ext4_zero_range(inode, offset, len, mode);
4624 WARN_ON_ONCE(!inode_is_locked(inode));
4625
4626 /* Indirect files do not support unwritten extents */
4627 if (!(ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS)))
4628 return -EOPNOTSUPP;
4629
4630 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
4631 (end > inode->i_size || end > EXT4_I(inode)->i_disksize)) {
4632 new_size = end;
4633 ret = inode_newsize_ok(inode, offset: new_size);
4634 if (ret)
4635 return ret;
4636 }
4637
4638 flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
4639 /* Preallocate the range including the unaligned edges */
4640 if (!IS_ALIGNED(offset | end, blocksize)) {
4641 ext4_lblk_t alloc_lblk = offset >> blkbits;
4642 ext4_lblk_t len_lblk = EXT4_MAX_BLOCKS(len, offset, blkbits);
4643
4644 ret = ext4_alloc_file_blocks(file, offset: alloc_lblk, len: len_lblk,
4645 new_size, flags);
4646 if (ret)
4647 return ret;
4648 }
4649
4650 ret = ext4_update_disksize_before_punch(inode, offset, len);
4651 if (ret)
4652 return ret;
4653
4654 /* Now release the pages and zero block aligned part of pages */
4655 ret = ext4_truncate_page_cache_block_range(inode, start: offset, end);
4656 if (ret)
4657 return ret;
4658
4659 /* Zero range excluding the unaligned edges */
4660 start_lblk = EXT4_B_TO_LBLK(inode, offset);
4661 end_lblk = end >> blkbits;
4662 if (end_lblk > start_lblk) {
4663 ext4_lblk_t zero_blks = end_lblk - start_lblk;
4664
4665 if (mode & FALLOC_FL_WRITE_ZEROES)
4666 flags = EXT4_GET_BLOCKS_CREATE_ZERO | EXT4_EX_NOCACHE;
4667 else
4668 flags |= (EXT4_GET_BLOCKS_CONVERT_UNWRITTEN |
4669 EXT4_EX_NOCACHE);
4670 ret = ext4_alloc_file_blocks(file, offset: start_lblk, len: zero_blks,
4671 new_size, flags);
4672 if (ret)
4673 return ret;
4674 }
4675 /* Finish zeroing out if it doesn't contain partial block */
4676 if (IS_ALIGNED(offset | end, blocksize))
4677 return ret;
4678
4679 /*
4680 * In worst case we have to writeout two nonadjacent unwritten
4681 * blocks and update the inode
4682 */
4683 credits = (2 * ext4_ext_index_trans_blocks(inode, extents: 2)) + 1;
4684 if (ext4_should_journal_data(inode))
4685 credits += 2;
4686 handle = ext4_journal_start(inode, EXT4_HT_MISC, credits);
4687 if (IS_ERR(ptr: handle)) {
4688 ret = PTR_ERR(ptr: handle);
4689 ext4_std_error(inode->i_sb, ret);
4690 return ret;
4691 }
4692
4693 /* Zero out partial block at the edges of the range */
4694 ret = ext4_zero_partial_blocks(handle, inode, lstart: offset, lend: len);
4695 if (ret)
4696 goto out_handle;
4697
4698 if (new_size)
4699 ext4_update_inode_size(inode, newsize: new_size);
4700 ret = ext4_mark_inode_dirty(handle, inode);
4701 if (unlikely(ret))
4702 goto out_handle;
4703
4704 ext4_update_inode_fsync_trans(handle, inode, datasync: 1);
4705 if (file->f_flags & O_SYNC)
4706 ext4_handle_sync(handle);
4707
4708out_handle:
4709 ext4_journal_stop(handle);
4710 return ret;
4711}
4712
4713static long ext4_do_fallocate(struct file *file, loff_t offset,
4714 loff_t len, int mode)
4715{
4716 struct inode *inode = file_inode(f: file);
4717 loff_t end = offset + len;
4718 loff_t new_size = 0;
4719 ext4_lblk_t start_lblk, len_lblk;
4720 int ret;
4721
4722 trace_ext4_fallocate_enter(inode, offset, len, mode);
4723 WARN_ON_ONCE(!inode_is_locked(inode));
4724
4725 start_lblk = offset >> inode->i_blkbits;
4726 len_lblk = EXT4_MAX_BLOCKS(len, offset, inode->i_blkbits);
4727
4728 /* We only support preallocation for extent-based files only. */
4729 if (!(ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))) {
4730 ret = -EOPNOTSUPP;
4731 goto out;
4732 }
4733
4734 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
4735 (end > inode->i_size || end > EXT4_I(inode)->i_disksize)) {
4736 new_size = end;
4737 ret = inode_newsize_ok(inode, offset: new_size);
4738 if (ret)
4739 goto out;
4740 }
4741
4742 ret = ext4_alloc_file_blocks(file, offset: start_lblk, len: len_lblk, new_size,
4743 EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT);
4744 if (ret)
4745 goto out;
4746
4747 if (file->f_flags & O_SYNC && EXT4_SB(sb: inode->i_sb)->s_journal) {
4748 ret = ext4_fc_commit(journal: EXT4_SB(sb: inode->i_sb)->s_journal,
4749 EXT4_I(inode)->i_sync_tid);
4750 }
4751out:
4752 trace_ext4_fallocate_exit(inode, offset, max_blocks: len_lblk, ret);
4753 return ret;
4754}
4755
4756/*
4757 * preallocate space for a file. This implements ext4's fallocate file
4758 * operation, which gets called from sys_fallocate system call.
4759 * For block-mapped files, posix_fallocate should fall back to the method
4760 * of writing zeroes to the required new blocks (the same behavior which is
4761 * expected for file systems which do not support fallocate() system call).
4762 */
4763long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
4764{
4765 struct inode *inode = file_inode(f: file);
4766 struct address_space *mapping = file->f_mapping;
4767 int ret;
4768
4769 /*
4770 * Encrypted inodes can't handle collapse range or insert
4771 * range since we would need to re-encrypt blocks with a
4772 * different IV or XTS tweak (which are based on the logical
4773 * block number).
4774 */
4775 if (IS_ENCRYPTED(inode) &&
4776 (mode & (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE)))
4777 return -EOPNOTSUPP;
4778 /*
4779 * Don't allow writing zeroes if the underlying device does not
4780 * enable the unmap write zeroes operation.
4781 */
4782 if ((mode & FALLOC_FL_WRITE_ZEROES) &&
4783 !bdev_write_zeroes_unmap_sectors(bdev: inode->i_sb->s_bdev))
4784 return -EOPNOTSUPP;
4785
4786 /* Return error if mode is not supported */
4787 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
4788 FALLOC_FL_ZERO_RANGE | FALLOC_FL_COLLAPSE_RANGE |
4789 FALLOC_FL_INSERT_RANGE | FALLOC_FL_WRITE_ZEROES))
4790 return -EOPNOTSUPP;
4791
4792 inode_lock(inode);
4793 ret = ext4_convert_inline_data(inode);
4794 if (ret)
4795 goto out_inode_lock;
4796
4797 /* Wait all existing dio workers, newcomers will block on i_rwsem */
4798 inode_dio_wait(inode);
4799
4800 ret = file_modified(file);
4801 if (ret)
4802 goto out_inode_lock;
4803
4804 if ((mode & FALLOC_FL_MODE_MASK) == FALLOC_FL_ALLOCATE_RANGE) {
4805 ret = ext4_do_fallocate(file, offset, len, mode);
4806 goto out_inode_lock;
4807 }
4808
4809 /*
4810 * Follow-up operations will drop page cache, hold invalidate lock
4811 * to prevent page faults from reinstantiating pages we have
4812 * released from page cache.
4813 */
4814 filemap_invalidate_lock(mapping);
4815
4816 ret = ext4_break_layouts(inode);
4817 if (ret)
4818 goto out_invalidate_lock;
4819
4820 switch (mode & FALLOC_FL_MODE_MASK) {
4821 case FALLOC_FL_PUNCH_HOLE:
4822 ret = ext4_punch_hole(file, offset, length: len);
4823 break;
4824 case FALLOC_FL_COLLAPSE_RANGE:
4825 ret = ext4_collapse_range(file, offset, len);
4826 break;
4827 case FALLOC_FL_INSERT_RANGE:
4828 ret = ext4_insert_range(file, offset, len);
4829 break;
4830 case FALLOC_FL_ZERO_RANGE:
4831 case FALLOC_FL_WRITE_ZEROES:
4832 ret = ext4_zero_range(file, offset, len, mode);
4833 break;
4834 default:
4835 ret = -EOPNOTSUPP;
4836 }
4837
4838out_invalidate_lock:
4839 filemap_invalidate_unlock(mapping);
4840out_inode_lock:
4841 inode_unlock(inode);
4842 return ret;
4843}
4844
4845/*
4846 * This function converts a range of blocks to written extents. The caller of
4847 * this function will pass the start offset and the size. all unwritten extents
4848 * within this range will be converted to written extents.
4849 *
4850 * This function is called from the direct IO end io call back function for
4851 * atomic writes, to convert the unwritten extents after IO is completed.
4852 *
4853 * Note that the requirement for atomic writes is that all conversion should
4854 * happen atomically in a single fs journal transaction. We mainly only allocate
4855 * unwritten extents either on a hole on a pre-exiting unwritten extent range in
4856 * ext4_map_blocks_atomic_write(). The only case where we can have multiple
4857 * unwritten extents in a range [offset, offset+len) is when there is a split
4858 * unwritten extent between two leaf nodes which was cached in extent status
4859 * cache during ext4_iomap_alloc() time. That will allow
4860 * ext4_map_blocks_atomic_write() to return the unwritten extent range w/o going
4861 * into the slow path. That means we might need a loop for conversion of this
4862 * unwritten extent split across leaf block within a single journal transaction.
4863 * Split extents across leaf nodes is a rare case, but let's still handle that
4864 * to meet the requirements of multi-fsblock atomic writes.
4865 *
4866 * Returns 0 on success.
4867 */
4868int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode,
4869 loff_t offset, ssize_t len)
4870{
4871 unsigned int max_blocks;
4872 int ret = 0, ret2 = 0, ret3 = 0;
4873 struct ext4_map_blocks map;
4874 unsigned int blkbits = inode->i_blkbits;
4875 unsigned int credits = 0;
4876 int flags = EXT4_GET_BLOCKS_IO_CONVERT_EXT | EXT4_EX_NOCACHE;
4877
4878 map.m_lblk = offset >> blkbits;
4879 max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);
4880
4881 if (!handle) {
4882 /*
4883 * TODO: An optimization can be added later by having an extent
4884 * status flag e.g. EXTENT_STATUS_SPLIT_LEAF. If we query that
4885 * it can tell if the extent in the cache is a split extent.
4886 * But for now let's assume pextents as 2 always.
4887 */
4888 credits = ext4_meta_trans_blocks(inode, lblocks: max_blocks, pextents: 2);
4889 }
4890
4891 if (credits) {
4892 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, credits);
4893 if (IS_ERR(ptr: handle)) {
4894 ret = PTR_ERR(ptr: handle);
4895 return ret;
4896 }
4897 }
4898
4899 while (ret >= 0 && ret < max_blocks) {
4900 map.m_lblk += ret;
4901 map.m_len = (max_blocks -= ret);
4902 ret = ext4_map_blocks(handle, inode, map: &map, flags);
4903 if (ret != max_blocks)
4904 ext4_msg(inode->i_sb, KERN_INFO,
4905 "inode #%lu: block %u: len %u: "
4906 "split block mapping found for atomic write, "
4907 "ret = %d",
4908 inode->i_ino, map.m_lblk,
4909 map.m_len, ret);
4910 if (ret <= 0)
4911 break;
4912 }
4913
4914 ret2 = ext4_mark_inode_dirty(handle, inode);
4915
4916 if (credits) {
4917 ret3 = ext4_journal_stop(handle);
4918 if (unlikely(ret3))
4919 ret2 = ret3;
4920 }
4921
4922 if (ret <= 0 || ret2)
4923 ext4_warning(inode->i_sb,
4924 "inode #%lu: block %u: len %u: "
4925 "returned %d or %d",
4926 inode->i_ino, map.m_lblk,
4927 map.m_len, ret, ret2);
4928
4929 return ret > 0 ? ret2 : ret;
4930}
4931
4932/*
4933 * This function convert a range of blocks to written extents
4934 * The caller of this function will pass the start offset and the size.
4935 * all unwritten extents within this range will be converted to
4936 * written extents.
4937 *
4938 * This function is called from the direct IO end io call back
4939 * function, to convert the fallocated extents after IO is completed.
4940 * Returns 0 on success.
4941 */
4942int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
4943 loff_t offset, ssize_t len)
4944{
4945 unsigned int max_blocks;
4946 int ret = 0, ret2 = 0, ret3 = 0;
4947 struct ext4_map_blocks map;
4948 unsigned int blkbits = inode->i_blkbits;
4949 unsigned int credits = 0;
4950
4951 map.m_lblk = offset >> blkbits;
4952 max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);
4953
4954 if (!handle) {
4955 /*
4956 * credits to insert 1 extent into extent tree
4957 */
4958 credits = ext4_chunk_trans_blocks(inode, nrblocks: max_blocks);
4959 }
4960 while (ret >= 0 && ret < max_blocks) {
4961 map.m_lblk += ret;
4962 map.m_len = (max_blocks -= ret);
4963 if (credits) {
4964 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
4965 credits);
4966 if (IS_ERR(ptr: handle)) {
4967 ret = PTR_ERR(ptr: handle);
4968 break;
4969 }
4970 }
4971 /*
4972 * Do not cache any unrelated extents, as it does not hold the
4973 * i_rwsem or invalidate_lock, which could corrupt the extent
4974 * status tree.
4975 */
4976 ret = ext4_map_blocks(handle, inode, map: &map,
4977 EXT4_GET_BLOCKS_IO_CONVERT_EXT |
4978 EXT4_EX_NOCACHE);
4979 if (ret <= 0)
4980 ext4_warning(inode->i_sb,
4981 "inode #%lu: block %u: len %u: "
4982 "ext4_ext_map_blocks returned %d",
4983 inode->i_ino, map.m_lblk,
4984 map.m_len, ret);
4985 ret2 = ext4_mark_inode_dirty(handle, inode);
4986 if (credits) {
4987 ret3 = ext4_journal_stop(handle);
4988 if (unlikely(ret3))
4989 ret2 = ret3;
4990 }
4991
4992 if (ret <= 0 || ret2)
4993 break;
4994 }
4995 return ret > 0 ? ret2 : ret;
4996}
4997
4998int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end)
4999{
5000 int ret = 0, err = 0;
5001 struct ext4_io_end_vec *io_end_vec;
5002
5003 /*
5004 * This is somewhat ugly but the idea is clear: When transaction is
5005 * reserved, everything goes into it. Otherwise we rather start several
5006 * smaller transactions for conversion of each extent separately.
5007 */
5008 if (handle) {
5009 handle = ext4_journal_start_reserved(handle,
5010 EXT4_HT_EXT_CONVERT);
5011 if (IS_ERR(ptr: handle))
5012 return PTR_ERR(ptr: handle);
5013 }
5014
5015 list_for_each_entry(io_end_vec, &io_end->list_vec, list) {
5016 ret = ext4_convert_unwritten_extents(handle, inode: io_end->inode,
5017 offset: io_end_vec->offset,
5018 len: io_end_vec->size);
5019 if (ret)
5020 break;
5021 }
5022
5023 if (handle)
5024 err = ext4_journal_stop(handle);
5025
5026 return ret < 0 ? ret : err;
5027}
5028
5029static int ext4_iomap_xattr_fiemap(struct inode *inode, struct iomap *iomap)
5030{
5031 __u64 physical = 0;
5032 __u64 length = 0;
5033 int blockbits = inode->i_sb->s_blocksize_bits;
5034 int error = 0;
5035 u16 iomap_type;
5036
5037 /* in-inode? */
5038 if (ext4_test_inode_state(inode, bit: EXT4_STATE_XATTR)) {
5039 struct ext4_iloc iloc;
5040 int offset; /* offset of xattr in inode */
5041
5042 error = ext4_get_inode_loc(inode, &iloc);
5043 if (error)
5044 return error;
5045 physical = (__u64)iloc.bh->b_blocknr << blockbits;
5046 offset = EXT4_GOOD_OLD_INODE_SIZE +
5047 EXT4_I(inode)->i_extra_isize;
5048 physical += offset;
5049 length = EXT4_SB(sb: inode->i_sb)->s_inode_size - offset;
5050 brelse(bh: iloc.bh);
5051 iomap_type = IOMAP_INLINE;
5052 } else if (EXT4_I(inode)->i_file_acl) { /* external block */
5053 physical = (__u64)EXT4_I(inode)->i_file_acl << blockbits;
5054 length = inode->i_sb->s_blocksize;
5055 iomap_type = IOMAP_MAPPED;
5056 } else {
5057 /* no in-inode or external block for xattr, so return -ENOENT */
5058 error = -ENOENT;
5059 goto out;
5060 }
5061
5062 iomap->addr = physical;
5063 iomap->offset = 0;
5064 iomap->length = length;
5065 iomap->type = iomap_type;
5066 iomap->flags = 0;
5067out:
5068 return error;
5069}
5070
5071static int ext4_iomap_xattr_begin(struct inode *inode, loff_t offset,
5072 loff_t length, unsigned flags,
5073 struct iomap *iomap, struct iomap *srcmap)
5074{
5075 int error;
5076
5077 error = ext4_iomap_xattr_fiemap(inode, iomap);
5078 if (error == 0 && (offset >= iomap->length))
5079 error = -ENOENT;
5080 return error;
5081}
5082
5083static const struct iomap_ops ext4_iomap_xattr_ops = {
5084 .iomap_begin = ext4_iomap_xattr_begin,
5085};
5086
5087static int ext4_fiemap_check_ranges(struct inode *inode, u64 start, u64 *len)
5088{
5089 u64 maxbytes = ext4_get_maxbytes(inode);
5090
5091 if (*len == 0)
5092 return -EINVAL;
5093 if (start > maxbytes)
5094 return -EFBIG;
5095
5096 /*
5097 * Shrink request scope to what the fs can actually handle.
5098 */
5099 if (*len > maxbytes || (maxbytes - *len) < start)
5100 *len = maxbytes - start;
5101 return 0;
5102}
5103
5104int ext4_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
5105 u64 start, u64 len)
5106{
5107 int error = 0;
5108
5109 inode_lock_shared(inode);
5110 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
5111 error = ext4_ext_precache(inode);
5112 if (error)
5113 goto unlock;
5114 fieinfo->fi_flags &= ~FIEMAP_FLAG_CACHE;
5115 }
5116
5117 /*
5118 * For bitmap files the maximum size limit could be smaller than
5119 * s_maxbytes, so check len here manually instead of just relying on the
5120 * generic check.
5121 */
5122 error = ext4_fiemap_check_ranges(inode, start, len: &len);
5123 if (error)
5124 goto unlock;
5125
5126 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
5127 fieinfo->fi_flags &= ~FIEMAP_FLAG_XATTR;
5128 error = iomap_fiemap(inode, fieinfo, start, len,
5129 ops: &ext4_iomap_xattr_ops);
5130 } else {
5131 error = iomap_fiemap(inode, fieinfo, start, len,
5132 ops: &ext4_iomap_report_ops);
5133 }
5134unlock:
5135 inode_unlock_shared(inode);
5136 return error;
5137}
5138
5139int ext4_get_es_cache(struct inode *inode, struct fiemap_extent_info *fieinfo,
5140 __u64 start, __u64 len)
5141{
5142 ext4_lblk_t start_blk, len_blks;
5143 __u64 last_blk;
5144 int error = 0;
5145
5146 if (ext4_has_inline_data(inode)) {
5147 int has_inline;
5148
5149 down_read(sem: &EXT4_I(inode)->xattr_sem);
5150 has_inline = ext4_has_inline_data(inode);
5151 up_read(sem: &EXT4_I(inode)->xattr_sem);
5152 if (has_inline)
5153 return 0;
5154 }
5155
5156 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
5157 inode_lock_shared(inode);
5158 error = ext4_ext_precache(inode);
5159 inode_unlock_shared(inode);
5160 if (error)
5161 return error;
5162 fieinfo->fi_flags &= ~FIEMAP_FLAG_CACHE;
5163 }
5164
5165 error = fiemap_prep(inode, fieinfo, start, len: &len, supported_flags: 0);
5166 if (error)
5167 return error;
5168
5169 error = ext4_fiemap_check_ranges(inode, start, len: &len);
5170 if (error)
5171 return error;
5172
5173 start_blk = start >> inode->i_sb->s_blocksize_bits;
5174 last_blk = (start + len - 1) >> inode->i_sb->s_blocksize_bits;
5175 if (last_blk >= EXT_MAX_BLOCKS)
5176 last_blk = EXT_MAX_BLOCKS-1;
5177 len_blks = ((ext4_lblk_t) last_blk) - start_blk + 1;
5178
5179 /*
5180 * Walk the extent tree gathering extent information
5181 * and pushing extents back to the user.
5182 */
5183 return ext4_fill_es_cache_info(inode, block: start_blk, num: len_blks, fieinfo);
5184}
5185
5186/*
5187 * ext4_ext_shift_path_extents:
5188 * Shift the extents of a path structure lying between path[depth].p_ext
5189 * and EXT_LAST_EXTENT(path[depth].p_hdr), by @shift blocks. @SHIFT tells
5190 * if it is right shift or left shift operation.
5191 */
5192static int
5193ext4_ext_shift_path_extents(struct ext4_ext_path *path, ext4_lblk_t shift,
5194 struct inode *inode, handle_t *handle,
5195 enum SHIFT_DIRECTION SHIFT)
5196{
5197 int depth, err = 0;
5198 struct ext4_extent *ex_start, *ex_last;
5199 bool update = false;
5200 int credits, restart_credits;
5201 depth = path->p_depth;
5202
5203 while (depth >= 0) {
5204 if (depth == path->p_depth) {
5205 ex_start = path[depth].p_ext;
5206 if (!ex_start)
5207 return -EFSCORRUPTED;
5208
5209 ex_last = EXT_LAST_EXTENT(path[depth].p_hdr);
5210 /* leaf + sb + inode */
5211 credits = 3;
5212 if (ex_start == EXT_FIRST_EXTENT(path[depth].p_hdr)) {
5213 update = true;
5214 /* extent tree + sb + inode */
5215 credits = depth + 2;
5216 }
5217
5218 restart_credits = ext4_chunk_trans_extent(inode, nrblocks: 0);
5219 err = ext4_datasem_ensure_credits(handle, inode, check_cred: credits,
5220 restart_cred: restart_credits, revoke_cred: 0);
5221 if (err) {
5222 if (err > 0)
5223 err = -EAGAIN;
5224 goto out;
5225 }
5226
5227 err = ext4_ext_get_access(handle, inode, path: path + depth);
5228 if (err)
5229 goto out;
5230
5231 while (ex_start <= ex_last) {
5232 if (SHIFT == SHIFT_LEFT) {
5233 le32_add_cpu(var: &ex_start->ee_block,
5234 val: -shift);
5235 /* Try to merge to the left. */
5236 if ((ex_start >
5237 EXT_FIRST_EXTENT(path[depth].p_hdr))
5238 &&
5239 ext4_ext_try_to_merge_right(inode,
5240 path, ex: ex_start - 1))
5241 ex_last--;
5242 else
5243 ex_start++;
5244 } else {
5245 le32_add_cpu(var: &ex_last->ee_block, val: shift);
5246 ext4_ext_try_to_merge_right(inode, path,
5247 ex: ex_last);
5248 ex_last--;
5249 }
5250 }
5251 err = ext4_ext_dirty(handle, inode, path + depth);
5252 if (err)
5253 goto out;
5254
5255 if (--depth < 0 || !update)
5256 break;
5257 }
5258
5259 /* Update index too */
5260 err = ext4_ext_get_access(handle, inode, path: path + depth);
5261 if (err)
5262 goto out;
5263
5264 if (SHIFT == SHIFT_LEFT)
5265 le32_add_cpu(var: &path[depth].p_idx->ei_block, val: -shift);
5266 else
5267 le32_add_cpu(var: &path[depth].p_idx->ei_block, val: shift);
5268 err = ext4_ext_dirty(handle, inode, path + depth);
5269 if (err)
5270 goto out;
5271
5272 /* we are done if current index is not a starting index */
5273 if (path[depth].p_idx != EXT_FIRST_INDEX(path[depth].p_hdr))
5274 break;
5275
5276 depth--;
5277 }
5278
5279out:
5280 return err;
5281}
5282
5283/*
5284 * ext4_ext_shift_extents:
5285 * All the extents which lies in the range from @start to the last allocated
5286 * block for the @inode are shifted either towards left or right (depending
5287 * upon @SHIFT) by @shift blocks.
5288 * On success, 0 is returned, error otherwise.
5289 */
5290static int
5291ext4_ext_shift_extents(struct inode *inode, handle_t *handle,
5292 ext4_lblk_t start, ext4_lblk_t shift,
5293 enum SHIFT_DIRECTION SHIFT)
5294{
5295 struct ext4_ext_path *path;
5296 int ret = 0, depth;
5297 struct ext4_extent *extent;
5298 ext4_lblk_t stop, *iterator, ex_start, ex_end;
5299 ext4_lblk_t tmp = EXT_MAX_BLOCKS;
5300
5301 /* Let path point to the last extent */
5302 path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL,
5303 EXT4_EX_NOCACHE);
5304 if (IS_ERR(ptr: path))
5305 return PTR_ERR(ptr: path);
5306
5307 depth = path->p_depth;
5308 extent = path[depth].p_ext;
5309 if (!extent)
5310 goto out;
5311
5312 stop = le32_to_cpu(extent->ee_block);
5313
5314 /*
5315 * For left shifts, make sure the hole on the left is big enough to
5316 * accommodate the shift. For right shifts, make sure the last extent
5317 * won't be shifted beyond EXT_MAX_BLOCKS.
5318 */
5319 if (SHIFT == SHIFT_LEFT) {
5320 path = ext4_find_extent(inode, block: start - 1, path,
5321 EXT4_EX_NOCACHE);
5322 if (IS_ERR(ptr: path))
5323 return PTR_ERR(ptr: path);
5324 depth = path->p_depth;
5325 extent = path[depth].p_ext;
5326 if (extent) {
5327 ex_start = le32_to_cpu(extent->ee_block);
5328 ex_end = le32_to_cpu(extent->ee_block) +
5329 ext4_ext_get_actual_len(ext: extent);
5330 } else {
5331 ex_start = 0;
5332 ex_end = 0;
5333 }
5334
5335 if ((start == ex_start && shift > ex_start) ||
5336 (shift > start - ex_end)) {
5337 ret = -EINVAL;
5338 goto out;
5339 }
5340 } else {
5341 if (shift > EXT_MAX_BLOCKS -
5342 (stop + ext4_ext_get_actual_len(ext: extent))) {
5343 ret = -EINVAL;
5344 goto out;
5345 }
5346 }
5347
5348 /*
5349 * In case of left shift, iterator points to start and it is increased
5350 * till we reach stop. In case of right shift, iterator points to stop
5351 * and it is decreased till we reach start.
5352 */
5353again:
5354 ret = 0;
5355 if (SHIFT == SHIFT_LEFT)
5356 iterator = &start;
5357 else
5358 iterator = &stop;
5359
5360 if (tmp != EXT_MAX_BLOCKS)
5361 *iterator = tmp;
5362
5363 /*
5364 * Its safe to start updating extents. Start and stop are unsigned, so
5365 * in case of right shift if extent with 0 block is reached, iterator
5366 * becomes NULL to indicate the end of the loop.
5367 */
5368 while (iterator && start <= stop) {
5369 path = ext4_find_extent(inode, block: *iterator, path,
5370 EXT4_EX_NOCACHE);
5371 if (IS_ERR(ptr: path))
5372 return PTR_ERR(ptr: path);
5373 depth = path->p_depth;
5374 extent = path[depth].p_ext;
5375 if (!extent) {
5376 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
5377 (unsigned long) *iterator);
5378 return -EFSCORRUPTED;
5379 }
5380 if (SHIFT == SHIFT_LEFT && *iterator >
5381 le32_to_cpu(extent->ee_block)) {
5382 /* Hole, move to the next extent */
5383 if (extent < EXT_LAST_EXTENT(path[depth].p_hdr)) {
5384 path[depth].p_ext++;
5385 } else {
5386 *iterator = ext4_ext_next_allocated_block(path);
5387 continue;
5388 }
5389 }
5390
5391 tmp = *iterator;
5392 if (SHIFT == SHIFT_LEFT) {
5393 extent = EXT_LAST_EXTENT(path[depth].p_hdr);
5394 *iterator = le32_to_cpu(extent->ee_block) +
5395 ext4_ext_get_actual_len(ext: extent);
5396 } else {
5397 extent = EXT_FIRST_EXTENT(path[depth].p_hdr);
5398 if (le32_to_cpu(extent->ee_block) > start)
5399 *iterator = le32_to_cpu(extent->ee_block) - 1;
5400 else if (le32_to_cpu(extent->ee_block) == start)
5401 iterator = NULL;
5402 else {
5403 extent = EXT_LAST_EXTENT(path[depth].p_hdr);
5404 while (le32_to_cpu(extent->ee_block) >= start)
5405 extent--;
5406
5407 if (extent == EXT_LAST_EXTENT(path[depth].p_hdr))
5408 break;
5409
5410 extent++;
5411 iterator = NULL;
5412 }
5413 path[depth].p_ext = extent;
5414 }
5415 ret = ext4_ext_shift_path_extents(path, shift, inode,
5416 handle, SHIFT);
5417 /* iterator can be NULL which means we should break */
5418 if (ret == -EAGAIN)
5419 goto again;
5420 if (ret)
5421 break;
5422 }
5423out:
5424 ext4_free_ext_path(path);
5425 return ret;
5426}
5427
5428/*
5429 * ext4_collapse_range:
5430 * This implements the fallocate's collapse range functionality for ext4
5431 * Returns: 0 and non-zero on error.
5432 */
5433static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len)
5434{
5435 struct inode *inode = file_inode(f: file);
5436 struct super_block *sb = inode->i_sb;
5437 struct address_space *mapping = inode->i_mapping;
5438 loff_t end = offset + len;
5439 ext4_lblk_t start_lblk, end_lblk;
5440 handle_t *handle;
5441 unsigned int credits;
5442 loff_t start, new_size;
5443 int ret;
5444
5445 trace_ext4_collapse_range(inode, offset, len);
5446 WARN_ON_ONCE(!inode_is_locked(inode));
5447
5448 /* Currently just for extent based files */
5449 if (!ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))
5450 return -EOPNOTSUPP;
5451 /* Collapse range works only on fs cluster size aligned regions. */
5452 if (!IS_ALIGNED(offset | len, EXT4_CLUSTER_SIZE(sb)))
5453 return -EINVAL;
5454 /*
5455 * There is no need to overlap collapse range with EOF, in which case
5456 * it is effectively a truncate operation
5457 */
5458 if (end >= inode->i_size)
5459 return -EINVAL;
5460
5461 /*
5462 * Write tail of the last page before removed range and data that
5463 * will be shifted since they will get removed from the page cache
5464 * below. We are also protected from pages becoming dirty by
5465 * i_rwsem and invalidate_lock.
5466 * Need to round down offset to be aligned with page size boundary
5467 * for page size > block size.
5468 */
5469 start = round_down(offset, PAGE_SIZE);
5470 ret = filemap_write_and_wait_range(mapping, lstart: start, lend: offset);
5471 if (!ret)
5472 ret = filemap_write_and_wait_range(mapping, lstart: end, LLONG_MAX);
5473 if (ret)
5474 return ret;
5475
5476 truncate_pagecache(inode, new: start);
5477
5478 credits = ext4_chunk_trans_extent(inode, nrblocks: 0);
5479 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
5480 if (IS_ERR(ptr: handle))
5481 return PTR_ERR(ptr: handle);
5482
5483 ext4_fc_mark_ineligible(sb, reason: EXT4_FC_REASON_FALLOC_RANGE, handle);
5484
5485 start_lblk = offset >> inode->i_blkbits;
5486 end_lblk = (offset + len) >> inode->i_blkbits;
5487
5488 ext4_check_map_extents_env(inode);
5489
5490 down_write(sem: &EXT4_I(inode)->i_data_sem);
5491 ext4_discard_preallocations(inode);
5492 ext4_es_remove_extent(inode, lblk: start_lblk, EXT_MAX_BLOCKS - start_lblk);
5493
5494 ret = ext4_ext_remove_space(inode, start: start_lblk, end: end_lblk - 1);
5495 if (ret) {
5496 up_write(sem: &EXT4_I(inode)->i_data_sem);
5497 goto out_handle;
5498 }
5499 ext4_discard_preallocations(inode);
5500
5501 ret = ext4_ext_shift_extents(inode, handle, start: end_lblk,
5502 shift: end_lblk - start_lblk, SHIFT: SHIFT_LEFT);
5503 if (ret) {
5504 up_write(sem: &EXT4_I(inode)->i_data_sem);
5505 goto out_handle;
5506 }
5507
5508 new_size = inode->i_size - len;
5509 i_size_write(inode, i_size: new_size);
5510 EXT4_I(inode)->i_disksize = new_size;
5511
5512 up_write(sem: &EXT4_I(inode)->i_data_sem);
5513 ret = ext4_mark_inode_dirty(handle, inode);
5514 if (ret)
5515 goto out_handle;
5516
5517 ext4_update_inode_fsync_trans(handle, inode, datasync: 1);
5518 if (IS_SYNC(inode))
5519 ext4_handle_sync(handle);
5520
5521out_handle:
5522 ext4_journal_stop(handle);
5523 return ret;
5524}
5525
5526/*
5527 * ext4_insert_range:
5528 * This function implements the FALLOC_FL_INSERT_RANGE flag of fallocate.
5529 * The data blocks starting from @offset to the EOF are shifted by @len
5530 * towards right to create a hole in the @inode. Inode size is increased
5531 * by len bytes.
5532 * Returns 0 on success, error otherwise.
5533 */
5534static int ext4_insert_range(struct file *file, loff_t offset, loff_t len)
5535{
5536 struct inode *inode = file_inode(f: file);
5537 struct super_block *sb = inode->i_sb;
5538 struct address_space *mapping = inode->i_mapping;
5539 handle_t *handle;
5540 struct ext4_ext_path *path;
5541 struct ext4_extent *extent;
5542 ext4_lblk_t start_lblk, len_lblk, ee_start_lblk = 0;
5543 unsigned int credits, ee_len;
5544 int ret, depth, split_flag = 0;
5545 loff_t start;
5546
5547 trace_ext4_insert_range(inode, offset, len);
5548 WARN_ON_ONCE(!inode_is_locked(inode));
5549
5550 /* Currently just for extent based files */
5551 if (!ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))
5552 return -EOPNOTSUPP;
5553 /* Insert range works only on fs cluster size aligned regions. */
5554 if (!IS_ALIGNED(offset | len, EXT4_CLUSTER_SIZE(sb)))
5555 return -EINVAL;
5556 /* Offset must be less than i_size */
5557 if (offset >= inode->i_size)
5558 return -EINVAL;
5559 /* Check whether the maximum file size would be exceeded */
5560 if (len > inode->i_sb->s_maxbytes - inode->i_size)
5561 return -EFBIG;
5562
5563 /*
5564 * Write out all dirty pages. Need to round down to align start offset
5565 * to page size boundary for page size > block size.
5566 */
5567 start = round_down(offset, PAGE_SIZE);
5568 ret = filemap_write_and_wait_range(mapping, lstart: start, LLONG_MAX);
5569 if (ret)
5570 return ret;
5571
5572 truncate_pagecache(inode, new: start);
5573
5574 credits = ext4_chunk_trans_extent(inode, nrblocks: 0);
5575 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
5576 if (IS_ERR(ptr: handle))
5577 return PTR_ERR(ptr: handle);
5578
5579 ext4_fc_mark_ineligible(sb, reason: EXT4_FC_REASON_FALLOC_RANGE, handle);
5580
5581 /* Expand file to avoid data loss if there is error while shifting */
5582 inode->i_size += len;
5583 EXT4_I(inode)->i_disksize += len;
5584 ret = ext4_mark_inode_dirty(handle, inode);
5585 if (ret)
5586 goto out_handle;
5587
5588 start_lblk = offset >> inode->i_blkbits;
5589 len_lblk = len >> inode->i_blkbits;
5590
5591 ext4_check_map_extents_env(inode);
5592
5593 down_write(sem: &EXT4_I(inode)->i_data_sem);
5594 ext4_discard_preallocations(inode);
5595
5596 path = ext4_find_extent(inode, block: start_lblk, NULL, flags: 0);
5597 if (IS_ERR(ptr: path)) {
5598 up_write(sem: &EXT4_I(inode)->i_data_sem);
5599 ret = PTR_ERR(ptr: path);
5600 goto out_handle;
5601 }
5602
5603 depth = ext_depth(inode);
5604 extent = path[depth].p_ext;
5605 if (extent) {
5606 ee_start_lblk = le32_to_cpu(extent->ee_block);
5607 ee_len = ext4_ext_get_actual_len(ext: extent);
5608
5609 /*
5610 * If start_lblk is not the starting block of extent, split
5611 * the extent @start_lblk
5612 */
5613 if ((start_lblk > ee_start_lblk) &&
5614 (start_lblk < (ee_start_lblk + ee_len))) {
5615 if (ext4_ext_is_unwritten(ext: extent))
5616 split_flag = EXT4_EXT_MARK_UNWRIT1 |
5617 EXT4_EXT_MARK_UNWRIT2;
5618 path = ext4_split_extent_at(handle, inode, path,
5619 split: start_lblk, split_flag,
5620 EXT4_EX_NOCACHE |
5621 EXT4_GET_BLOCKS_PRE_IO |
5622 EXT4_GET_BLOCKS_METADATA_NOFAIL);
5623 }
5624
5625 if (IS_ERR(ptr: path)) {
5626 up_write(sem: &EXT4_I(inode)->i_data_sem);
5627 ret = PTR_ERR(ptr: path);
5628 goto out_handle;
5629 }
5630 }
5631
5632 ext4_free_ext_path(path);
5633 ext4_es_remove_extent(inode, lblk: start_lblk, EXT_MAX_BLOCKS - start_lblk);
5634
5635 /*
5636 * if start_lblk lies in a hole which is at start of file, use
5637 * ee_start_lblk to shift extents
5638 */
5639 ret = ext4_ext_shift_extents(inode, handle,
5640 max(ee_start_lblk, start_lblk), shift: len_lblk, SHIFT: SHIFT_RIGHT);
5641 up_write(sem: &EXT4_I(inode)->i_data_sem);
5642 if (ret)
5643 goto out_handle;
5644
5645 ext4_update_inode_fsync_trans(handle, inode, datasync: 1);
5646 if (IS_SYNC(inode))
5647 ext4_handle_sync(handle);
5648
5649out_handle:
5650 ext4_journal_stop(handle);
5651 return ret;
5652}
5653
5654/**
5655 * ext4_swap_extents() - Swap extents between two inodes
5656 * @handle: handle for this transaction
5657 * @inode1: First inode
5658 * @inode2: Second inode
5659 * @lblk1: Start block for first inode
5660 * @lblk2: Start block for second inode
5661 * @count: Number of blocks to swap
5662 * @unwritten: Mark second inode's extents as unwritten after swap
5663 * @erp: Pointer to save error value
5664 *
5665 * This helper routine does exactly what is promise "swap extents". All other
5666 * stuff such as page-cache locking consistency, bh mapping consistency or
5667 * extent's data copying must be performed by caller.
5668 * Locking:
5669 * i_rwsem is held for both inodes
5670 * i_data_sem is locked for write for both inodes
5671 * Assumptions:
5672 * All pages from requested range are locked for both inodes
5673 */
5674int
5675ext4_swap_extents(handle_t *handle, struct inode *inode1,
5676 struct inode *inode2, ext4_lblk_t lblk1, ext4_lblk_t lblk2,
5677 ext4_lblk_t count, int unwritten, int *erp)
5678{
5679 struct ext4_ext_path *path1 = NULL;
5680 struct ext4_ext_path *path2 = NULL;
5681 int replaced_count = 0;
5682
5683 BUG_ON(!rwsem_is_locked(&EXT4_I(inode1)->i_data_sem));
5684 BUG_ON(!rwsem_is_locked(&EXT4_I(inode2)->i_data_sem));
5685 BUG_ON(!inode_is_locked(inode1));
5686 BUG_ON(!inode_is_locked(inode2));
5687
5688 ext4_es_remove_extent(inode: inode1, lblk: lblk1, len: count);
5689 ext4_es_remove_extent(inode: inode2, lblk: lblk2, len: count);
5690
5691 while (count) {
5692 struct ext4_extent *ex1, *ex2, tmp_ex;
5693 ext4_lblk_t e1_blk, e2_blk;
5694 int e1_len, e2_len, len;
5695 int split = 0;
5696
5697 path1 = ext4_find_extent(inode: inode1, block: lblk1, path: path1, EXT4_EX_NOCACHE);
5698 if (IS_ERR(ptr: path1)) {
5699 *erp = PTR_ERR(ptr: path1);
5700 goto errout;
5701 }
5702 path2 = ext4_find_extent(inode: inode2, block: lblk2, path: path2, EXT4_EX_NOCACHE);
5703 if (IS_ERR(ptr: path2)) {
5704 *erp = PTR_ERR(ptr: path2);
5705 goto errout;
5706 }
5707 ex1 = path1[path1->p_depth].p_ext;
5708 ex2 = path2[path2->p_depth].p_ext;
5709 /* Do we have something to swap ? */
5710 if (unlikely(!ex2 || !ex1))
5711 goto errout;
5712
5713 e1_blk = le32_to_cpu(ex1->ee_block);
5714 e2_blk = le32_to_cpu(ex2->ee_block);
5715 e1_len = ext4_ext_get_actual_len(ext: ex1);
5716 e2_len = ext4_ext_get_actual_len(ext: ex2);
5717
5718 /* Hole handling */
5719 if (!in_range(lblk1, e1_blk, e1_len) ||
5720 !in_range(lblk2, e2_blk, e2_len)) {
5721 ext4_lblk_t next1, next2;
5722
5723 /* if hole after extent, then go to next extent */
5724 next1 = ext4_ext_next_allocated_block(path: path1);
5725 next2 = ext4_ext_next_allocated_block(path: path2);
5726 /* If hole before extent, then shift to that extent */
5727 if (e1_blk > lblk1)
5728 next1 = e1_blk;
5729 if (e2_blk > lblk2)
5730 next2 = e2_blk;
5731 /* Do we have something to swap */
5732 if (next1 == EXT_MAX_BLOCKS || next2 == EXT_MAX_BLOCKS)
5733 goto errout;
5734 /* Move to the rightest boundary */
5735 len = next1 - lblk1;
5736 if (len < next2 - lblk2)
5737 len = next2 - lblk2;
5738 if (len > count)
5739 len = count;
5740 lblk1 += len;
5741 lblk2 += len;
5742 count -= len;
5743 continue;
5744 }
5745
5746 /* Prepare left boundary */
5747 if (e1_blk < lblk1) {
5748 split = 1;
5749 path1 = ext4_force_split_extent_at(handle, inode: inode1,
5750 path: path1, lblk: lblk1, nofail: 0);
5751 if (IS_ERR(ptr: path1)) {
5752 *erp = PTR_ERR(ptr: path1);
5753 goto errout;
5754 }
5755 }
5756 if (e2_blk < lblk2) {
5757 split = 1;
5758 path2 = ext4_force_split_extent_at(handle, inode: inode2,
5759 path: path2, lblk: lblk2, nofail: 0);
5760 if (IS_ERR(ptr: path2)) {
5761 *erp = PTR_ERR(ptr: path2);
5762 goto errout;
5763 }
5764 }
5765 /* ext4_split_extent_at() may result in leaf extent split,
5766 * path must to be revalidated. */
5767 if (split)
5768 continue;
5769
5770 /* Prepare right boundary */
5771 len = count;
5772 if (len > e1_blk + e1_len - lblk1)
5773 len = e1_blk + e1_len - lblk1;
5774 if (len > e2_blk + e2_len - lblk2)
5775 len = e2_blk + e2_len - lblk2;
5776
5777 if (len != e1_len) {
5778 split = 1;
5779 path1 = ext4_force_split_extent_at(handle, inode: inode1,
5780 path: path1, lblk: lblk1 + len, nofail: 0);
5781 if (IS_ERR(ptr: path1)) {
5782 *erp = PTR_ERR(ptr: path1);
5783 goto errout;
5784 }
5785 }
5786 if (len != e2_len) {
5787 split = 1;
5788 path2 = ext4_force_split_extent_at(handle, inode: inode2,
5789 path: path2, lblk: lblk2 + len, nofail: 0);
5790 if (IS_ERR(ptr: path2)) {
5791 *erp = PTR_ERR(ptr: path2);
5792 goto errout;
5793 }
5794 }
5795 /* ext4_split_extent_at() may result in leaf extent split,
5796 * path must to be revalidated. */
5797 if (split)
5798 continue;
5799
5800 BUG_ON(e2_len != e1_len);
5801 *erp = ext4_ext_get_access(handle, inode: inode1, path: path1 + path1->p_depth);
5802 if (unlikely(*erp))
5803 goto errout;
5804 *erp = ext4_ext_get_access(handle, inode: inode2, path: path2 + path2->p_depth);
5805 if (unlikely(*erp))
5806 goto errout;
5807
5808 /* Both extents are fully inside boundaries. Swap it now */
5809 tmp_ex = *ex1;
5810 ext4_ext_store_pblock(ex: ex1, pb: ext4_ext_pblock(ex: ex2));
5811 ext4_ext_store_pblock(ex: ex2, pb: ext4_ext_pblock(ex: &tmp_ex));
5812 ex1->ee_len = cpu_to_le16(e2_len);
5813 ex2->ee_len = cpu_to_le16(e1_len);
5814 if (unwritten)
5815 ext4_ext_mark_unwritten(ext: ex2);
5816 if (ext4_ext_is_unwritten(ext: &tmp_ex))
5817 ext4_ext_mark_unwritten(ext: ex1);
5818
5819 ext4_ext_try_to_merge(handle, inode: inode2, path: path2, ex: ex2);
5820 ext4_ext_try_to_merge(handle, inode: inode1, path: path1, ex: ex1);
5821 *erp = ext4_ext_dirty(handle, inode2, path2 +
5822 path2->p_depth);
5823 if (unlikely(*erp))
5824 goto errout;
5825 *erp = ext4_ext_dirty(handle, inode1, path1 +
5826 path1->p_depth);
5827 /*
5828 * Looks scarry ah..? second inode already points to new blocks,
5829 * and it was successfully dirtied. But luckily error may happen
5830 * only due to journal error, so full transaction will be
5831 * aborted anyway.
5832 */
5833 if (unlikely(*erp))
5834 goto errout;
5835
5836 lblk1 += len;
5837 lblk2 += len;
5838 replaced_count += len;
5839 count -= len;
5840 }
5841
5842errout:
5843 ext4_free_ext_path(path: path1);
5844 ext4_free_ext_path(path: path2);
5845 return replaced_count;
5846}
5847
5848/*
5849 * ext4_clu_mapped - determine whether any block in a logical cluster has
5850 * been mapped to a physical cluster
5851 *
5852 * @inode - file containing the logical cluster
5853 * @lclu - logical cluster of interest
5854 *
5855 * Returns 1 if any block in the logical cluster is mapped, signifying
5856 * that a physical cluster has been allocated for it. Otherwise,
5857 * returns 0. Can also return negative error codes. Derived from
5858 * ext4_ext_map_blocks().
5859 */
5860int ext4_clu_mapped(struct inode *inode, ext4_lblk_t lclu)
5861{
5862 struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
5863 struct ext4_ext_path *path;
5864 int depth, mapped = 0, err = 0;
5865 struct ext4_extent *extent;
5866 ext4_lblk_t first_lblk, first_lclu, last_lclu;
5867
5868 /*
5869 * if data can be stored inline, the logical cluster isn't
5870 * mapped - no physical clusters have been allocated, and the
5871 * file has no extents
5872 */
5873 if (ext4_test_inode_state(inode, bit: EXT4_STATE_MAY_INLINE_DATA) ||
5874 ext4_has_inline_data(inode))
5875 return 0;
5876
5877 /* search for the extent closest to the first block in the cluster */
5878 path = ext4_find_extent(inode, EXT4_C2B(sbi, lclu), NULL, flags: 0);
5879 if (IS_ERR(ptr: path))
5880 return PTR_ERR(ptr: path);
5881
5882 depth = ext_depth(inode);
5883
5884 /*
5885 * A consistent leaf must not be empty. This situation is possible,
5886 * though, _during_ tree modification, and it's why an assert can't
5887 * be put in ext4_find_extent().
5888 */
5889 if (unlikely(path[depth].p_ext == NULL && depth != 0)) {
5890 EXT4_ERROR_INODE(inode,
5891 "bad extent address - lblock: %lu, depth: %d, pblock: %lld",
5892 (unsigned long) EXT4_C2B(sbi, lclu),
5893 depth, path[depth].p_block);
5894 err = -EFSCORRUPTED;
5895 goto out;
5896 }
5897
5898 extent = path[depth].p_ext;
5899
5900 /* can't be mapped if the extent tree is empty */
5901 if (extent == NULL)
5902 goto out;
5903
5904 first_lblk = le32_to_cpu(extent->ee_block);
5905 first_lclu = EXT4_B2C(sbi, first_lblk);
5906
5907 /*
5908 * Three possible outcomes at this point - found extent spanning
5909 * the target cluster, to the left of the target cluster, or to the
5910 * right of the target cluster. The first two cases are handled here.
5911 * The last case indicates the target cluster is not mapped.
5912 */
5913 if (lclu >= first_lclu) {
5914 last_lclu = EXT4_B2C(sbi, first_lblk +
5915 ext4_ext_get_actual_len(extent) - 1);
5916 if (lclu <= last_lclu) {
5917 mapped = 1;
5918 } else {
5919 first_lblk = ext4_ext_next_allocated_block(path);
5920 first_lclu = EXT4_B2C(sbi, first_lblk);
5921 if (lclu == first_lclu)
5922 mapped = 1;
5923 }
5924 }
5925
5926out:
5927 ext4_free_ext_path(path);
5928
5929 return err ? err : mapped;
5930}
5931
5932/*
5933 * Updates physical block address and unwritten status of extent
5934 * starting at lblk start and of len. If such an extent doesn't exist,
5935 * this function splits the extent tree appropriately to create an
5936 * extent like this. This function is called in the fast commit
5937 * replay path. Returns 0 on success and error on failure.
5938 */
5939int ext4_ext_replay_update_ex(struct inode *inode, ext4_lblk_t start,
5940 int len, int unwritten, ext4_fsblk_t pblk)
5941{
5942 struct ext4_ext_path *path;
5943 struct ext4_extent *ex;
5944 int ret;
5945
5946 path = ext4_find_extent(inode, block: start, NULL, flags: 0);
5947 if (IS_ERR(ptr: path))
5948 return PTR_ERR(ptr: path);
5949 ex = path[path->p_depth].p_ext;
5950 if (!ex) {
5951 ret = -EFSCORRUPTED;
5952 goto out;
5953 }
5954
5955 if (le32_to_cpu(ex->ee_block) != start ||
5956 ext4_ext_get_actual_len(ext: ex) != len) {
5957 /* We need to split this extent to match our extent first */
5958 down_write(sem: &EXT4_I(inode)->i_data_sem);
5959 path = ext4_force_split_extent_at(NULL, inode, path, lblk: start, nofail: 1);
5960 up_write(sem: &EXT4_I(inode)->i_data_sem);
5961 if (IS_ERR(ptr: path)) {
5962 ret = PTR_ERR(ptr: path);
5963 goto out;
5964 }
5965
5966 path = ext4_find_extent(inode, block: start, path, flags: 0);
5967 if (IS_ERR(ptr: path))
5968 return PTR_ERR(ptr: path);
5969
5970 ex = path[path->p_depth].p_ext;
5971 WARN_ON(le32_to_cpu(ex->ee_block) != start);
5972
5973 if (ext4_ext_get_actual_len(ext: ex) != len) {
5974 down_write(sem: &EXT4_I(inode)->i_data_sem);
5975 path = ext4_force_split_extent_at(NULL, inode, path,
5976 lblk: start + len, nofail: 1);
5977 up_write(sem: &EXT4_I(inode)->i_data_sem);
5978 if (IS_ERR(ptr: path)) {
5979 ret = PTR_ERR(ptr: path);
5980 goto out;
5981 }
5982
5983 path = ext4_find_extent(inode, block: start, path, flags: 0);
5984 if (IS_ERR(ptr: path))
5985 return PTR_ERR(ptr: path);
5986 ex = path[path->p_depth].p_ext;
5987 }
5988 }
5989 if (unwritten)
5990 ext4_ext_mark_unwritten(ext: ex);
5991 else
5992 ext4_ext_mark_initialized(ext: ex);
5993 ext4_ext_store_pblock(ex, pb: pblk);
5994 down_write(sem: &EXT4_I(inode)->i_data_sem);
5995 ret = ext4_ext_dirty(NULL, inode, &path[path->p_depth]);
5996 up_write(sem: &EXT4_I(inode)->i_data_sem);
5997out:
5998 ext4_free_ext_path(path);
5999 ext4_mark_inode_dirty(NULL, inode);
6000 return ret;
6001}
6002
6003/* Try to shrink the extent tree */
6004void ext4_ext_replay_shrink_inode(struct inode *inode, ext4_lblk_t end)
6005{
6006 struct ext4_ext_path *path = NULL;
6007 struct ext4_extent *ex;
6008 ext4_lblk_t old_cur, cur = 0;
6009
6010 while (cur < end) {
6011 path = ext4_find_extent(inode, block: cur, NULL, flags: 0);
6012 if (IS_ERR(ptr: path))
6013 return;
6014 ex = path[path->p_depth].p_ext;
6015 if (!ex) {
6016 ext4_free_ext_path(path);
6017 ext4_mark_inode_dirty(NULL, inode);
6018 return;
6019 }
6020 old_cur = cur;
6021 cur = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ext: ex);
6022 if (cur <= old_cur)
6023 cur = old_cur + 1;
6024 ext4_ext_try_to_merge(NULL, inode, path, ex);
6025 down_write(sem: &EXT4_I(inode)->i_data_sem);
6026 ext4_ext_dirty(NULL, inode, &path[path->p_depth]);
6027 up_write(sem: &EXT4_I(inode)->i_data_sem);
6028 ext4_mark_inode_dirty(NULL, inode);
6029 ext4_free_ext_path(path);
6030 }
6031}
6032
6033/* Check if *cur is a hole and if it is, skip it */
6034static int skip_hole(struct inode *inode, ext4_lblk_t *cur)
6035{
6036 int ret;
6037 struct ext4_map_blocks map;
6038
6039 map.m_lblk = *cur;
6040 map.m_len = ((inode->i_size) >> inode->i_sb->s_blocksize_bits) - *cur;
6041
6042 ret = ext4_map_blocks(NULL, inode, map: &map, flags: 0);
6043 if (ret < 0)
6044 return ret;
6045 if (ret != 0)
6046 return 0;
6047 *cur = *cur + map.m_len;
6048 return 0;
6049}
6050
6051/* Count number of blocks used by this inode and update i_blocks */
6052int ext4_ext_replay_set_iblocks(struct inode *inode)
6053{
6054 struct ext4_ext_path *path = NULL, *path2 = NULL;
6055 struct ext4_extent *ex;
6056 ext4_lblk_t cur = 0, end;
6057 int numblks = 0, i, ret = 0;
6058 ext4_fsblk_t cmp1, cmp2;
6059 struct ext4_map_blocks map;
6060
6061 /* Determin the size of the file first */
6062 path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL,
6063 EXT4_EX_NOCACHE);
6064 if (IS_ERR(ptr: path))
6065 return PTR_ERR(ptr: path);
6066 ex = path[path->p_depth].p_ext;
6067 if (!ex)
6068 goto out;
6069 end = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ext: ex);
6070
6071 /* Count the number of data blocks */
6072 cur = 0;
6073 while (cur < end) {
6074 map.m_lblk = cur;
6075 map.m_len = end - cur;
6076 ret = ext4_map_blocks(NULL, inode, map: &map, flags: 0);
6077 if (ret < 0)
6078 break;
6079 if (ret > 0)
6080 numblks += ret;
6081 cur = cur + map.m_len;
6082 }
6083
6084 /*
6085 * Count the number of extent tree blocks. We do it by looking up
6086 * two successive extents and determining the difference between
6087 * their paths. When path is different for 2 successive extents
6088 * we compare the blocks in the path at each level and increment
6089 * iblocks by total number of differences found.
6090 */
6091 cur = 0;
6092 ret = skip_hole(inode, cur: &cur);
6093 if (ret < 0)
6094 goto out;
6095 path = ext4_find_extent(inode, block: cur, path, flags: 0);
6096 if (IS_ERR(ptr: path))
6097 goto out;
6098 numblks += path->p_depth;
6099 while (cur < end) {
6100 path = ext4_find_extent(inode, block: cur, path, flags: 0);
6101 if (IS_ERR(ptr: path))
6102 break;
6103 ex = path[path->p_depth].p_ext;
6104 if (!ex)
6105 goto cleanup;
6106
6107 cur = max(cur + 1, le32_to_cpu(ex->ee_block) +
6108 ext4_ext_get_actual_len(ex));
6109 ret = skip_hole(inode, cur: &cur);
6110 if (ret < 0)
6111 break;
6112
6113 path2 = ext4_find_extent(inode, block: cur, path: path2, flags: 0);
6114 if (IS_ERR(ptr: path2))
6115 break;
6116
6117 for (i = 0; i <= max(path->p_depth, path2->p_depth); i++) {
6118 cmp1 = cmp2 = 0;
6119 if (i <= path->p_depth)
6120 cmp1 = path[i].p_bh ?
6121 path[i].p_bh->b_blocknr : 0;
6122 if (i <= path2->p_depth)
6123 cmp2 = path2[i].p_bh ?
6124 path2[i].p_bh->b_blocknr : 0;
6125 if (cmp1 != cmp2 && cmp2 != 0)
6126 numblks++;
6127 }
6128 }
6129
6130out:
6131 inode->i_blocks = numblks << (inode->i_sb->s_blocksize_bits - 9);
6132 ext4_mark_inode_dirty(NULL, inode);
6133cleanup:
6134 ext4_free_ext_path(path);
6135 ext4_free_ext_path(path: path2);
6136 return 0;
6137}
6138
6139int ext4_ext_clear_bb(struct inode *inode)
6140{
6141 struct ext4_ext_path *path = NULL;
6142 struct ext4_extent *ex;
6143 ext4_lblk_t cur = 0, end;
6144 int j, ret = 0;
6145 struct ext4_map_blocks map;
6146
6147 if (ext4_test_inode_flag(inode, bit: EXT4_INODE_INLINE_DATA))
6148 return 0;
6149
6150 /* Determin the size of the file first */
6151 path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL,
6152 EXT4_EX_NOCACHE);
6153 if (IS_ERR(ptr: path))
6154 return PTR_ERR(ptr: path);
6155 ex = path[path->p_depth].p_ext;
6156 if (!ex)
6157 goto out;
6158 end = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ext: ex);
6159
6160 cur = 0;
6161 while (cur < end) {
6162 map.m_lblk = cur;
6163 map.m_len = end - cur;
6164 ret = ext4_map_blocks(NULL, inode, map: &map, flags: 0);
6165 if (ret < 0)
6166 break;
6167 if (ret > 0) {
6168 path = ext4_find_extent(inode, block: map.m_lblk, path, flags: 0);
6169 if (!IS_ERR(ptr: path)) {
6170 for (j = 0; j < path->p_depth; j++) {
6171 ext4_mb_mark_bb(sb: inode->i_sb,
6172 block: path[j].p_block, len: 1, state: false);
6173 ext4_fc_record_regions(sb: inode->i_sb, ino: inode->i_ino,
6174 lblk: 0, pblk: path[j].p_block, len: 1, replay: 1);
6175 }
6176 } else {
6177 path = NULL;
6178 }
6179 ext4_mb_mark_bb(sb: inode->i_sb, block: map.m_pblk, len: map.m_len, state: false);
6180 ext4_fc_record_regions(sb: inode->i_sb, ino: inode->i_ino,
6181 lblk: map.m_lblk, pblk: map.m_pblk, len: map.m_len, replay: 1);
6182 }
6183 cur = cur + map.m_len;
6184 }
6185
6186out:
6187 ext4_free_ext_path(path);
6188 return 0;
6189}
6190