1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * fs/ext4/fast_commit.c
5 *
6 * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com>
7 *
8 * Ext4 fast commits routines.
9 */
10#include "ext4.h"
11#include "ext4_jbd2.h"
12#include "ext4_extents.h"
13#include "mballoc.h"
14
15#include <linux/lockdep.h>
16/*
17 * Ext4 Fast Commits
18 * -----------------
19 *
20 * Ext4 fast commits implement fine grained journalling for Ext4.
21 *
22 * Fast commits are organized as a log of tag-length-value (TLV) structs. (See
23 * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by
24 * TLV during the recovery phase. For the scenarios for which we currently
25 * don't have replay code, fast commit falls back to full commits.
26 * Fast commits record delta in one of the following three categories.
27 *
28 * (A) Directory entry updates:
29 *
30 * - EXT4_FC_TAG_UNLINK - records directory entry unlink
31 * - EXT4_FC_TAG_LINK - records directory entry link
32 * - EXT4_FC_TAG_CREAT - records inode and directory entry creation
33 *
34 * (B) File specific data range updates:
35 *
36 * - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode
37 * - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode
38 *
39 * (C) Inode metadata (mtime / ctime etc):
40 *
41 * - EXT4_FC_TAG_INODE - record the inode that should be replayed
42 * during recovery. Note that iblocks field is
43 * not replayed and instead derived during
44 * replay.
45 * Commit Operation
46 * ----------------
47 * With fast commits, we maintain all the directory entry operations in the
48 * order in which they are issued in an in-memory queue. This queue is flushed
49 * to disk during the commit operation. We also maintain a list of inodes
50 * that need to be committed during a fast commit in another in memory queue of
51 * inodes. During the commit operation, we commit in the following order:
52 *
53 * [1] Prepare all the inodes to write out their data by setting
54 * "EXT4_STATE_FC_FLUSHING_DATA". This ensures that inode cannot be
55 * deleted while it is being flushed.
56 * [2] Flush data buffers to disk and clear "EXT4_STATE_FC_FLUSHING_DATA"
57 * state.
58 * [3] Lock the journal by calling jbd2_journal_lock_updates. This ensures that
59 * all the exsiting handles finish and no new handles can start.
60 * [4] Mark all the fast commit eligible inodes as undergoing fast commit
61 * by setting "EXT4_STATE_FC_COMMITTING" state.
62 * [5] Unlock the journal by calling jbd2_journal_unlock_updates. This allows
63 * starting of new handles. If new handles try to start an update on
64 * any of the inodes that are being committed, ext4_fc_track_inode()
65 * will block until those inodes have finished the fast commit.
66 * [6] Commit all the directory entry updates in the fast commit space.
67 * [7] Commit all the changed inodes in the fast commit space and clear
68 * "EXT4_STATE_FC_COMMITTING" for these inodes.
69 * [8] Write tail tag (this tag ensures the atomicity, please read the following
70 * section for more details).
71 *
72 * All the inode updates must be enclosed within jbd2_jounrnal_start()
73 * and jbd2_journal_stop() similar to JBD2 journaling.
74 *
75 * Fast Commit Ineligibility
76 * -------------------------
77 *
78 * Not all operations are supported by fast commits today (e.g extended
79 * attributes). Fast commit ineligibility is marked by calling
80 * ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back
81 * to full commit.
82 *
83 * Atomicity of commits
84 * --------------------
85 * In order to guarantee atomicity during the commit operation, fast commit
86 * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail
87 * tag contains CRC of the contents and TID of the transaction after which
88 * this fast commit should be applied. Recovery code replays fast commit
89 * logs only if there's at least 1 valid tail present. For every fast commit
90 * operation, there is 1 tail. This means, we may end up with multiple tails
91 * in the fast commit space. Here's an example:
92 *
93 * - Create a new file A and remove existing file B
94 * - fsync()
95 * - Append contents to file A
96 * - Truncate file A
97 * - fsync()
98 *
99 * The fast commit space at the end of above operations would look like this:
100 * [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL]
101 * |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->|
102 *
103 * Replay code should thus check for all the valid tails in the FC area.
104 *
105 * Fast Commit Replay Idempotence
106 * ------------------------------
107 *
108 * Fast commits tags are idempotent in nature provided the recovery code follows
109 * certain rules. The guiding principle that the commit path follows while
110 * committing is that it stores the result of a particular operation instead of
111 * storing the procedure.
112 *
113 * Let's consider this rename operation: 'mv /a /b'. Let's assume dirent '/a'
114 * was associated with inode 10. During fast commit, instead of storing this
115 * operation as a procedure "rename a to b", we store the resulting file system
116 * state as a "series" of outcomes:
117 *
118 * - Link dirent b to inode 10
119 * - Unlink dirent a
120 * - Inode <10> with valid refcount
121 *
122 * Now when recovery code runs, it needs "enforce" this state on the file
123 * system. This is what guarantees idempotence of fast commit replay.
124 *
125 * Let's take an example of a procedure that is not idempotent and see how fast
126 * commits make it idempotent. Consider following sequence of operations:
127 *
128 * rm A; mv B A; read A
129 * (x) (y) (z)
130 *
131 * (x), (y) and (z) are the points at which we can crash. If we store this
132 * sequence of operations as is then the replay is not idempotent. Let's say
133 * while in replay, we crash at (z). During the second replay, file A (which was
134 * actually created as a result of "mv B A" operation) would get deleted. Thus,
135 * file named A would be absent when we try to read A. So, this sequence of
136 * operations is not idempotent. However, as mentioned above, instead of storing
137 * the procedure fast commits store the outcome of each procedure. Thus the fast
138 * commit log for above procedure would be as follows:
139 *
140 * (Let's assume dirent A was linked to inode 10 and dirent B was linked to
141 * inode 11 before the replay)
142 *
143 * [Unlink A] [Link A to inode 11] [Unlink B] [Inode 11]
144 * (w) (x) (y) (z)
145 *
146 * If we crash at (z), we will have file A linked to inode 11. During the second
147 * replay, we will remove file A (inode 11). But we will create it back and make
148 * it point to inode 11. We won't find B, so we'll just skip that step. At this
149 * point, the refcount for inode 11 is not reliable, but that gets fixed by the
150 * replay of last inode 11 tag. Crashes at points (w), (x) and (y) get handled
151 * similarly. Thus, by converting a non-idempotent procedure into a series of
152 * idempotent outcomes, fast commits ensured idempotence during the replay.
153 *
154 * Locking
155 * -------
156 * sbi->s_fc_lock protects the fast commit inodes queue and the fast commit
157 * dentry queue. ei->i_fc_lock protects the fast commit related info in a given
158 * inode. Most of the code avoids acquiring both the locks, but if one must do
159 * that then sbi->s_fc_lock must be acquired before ei->i_fc_lock.
160 *
161 * TODOs
162 * -----
163 *
164 * 0) Fast commit replay path hardening: Fast commit replay code should use
165 * journal handles to make sure all the updates it does during the replay
166 * path are atomic. With that if we crash during fast commit replay, after
167 * trying to do recovery again, we will find a file system where fast commit
168 * area is invalid (because new full commit would be found). In order to deal
169 * with that, fast commit replay code should ensure that the "FC_REPLAY"
170 * superblock state is persisted before starting the replay, so that after
171 * the crash, fast commit recovery code can look at that flag and perform
172 * fast commit recovery even if that area is invalidated by later full
173 * commits.
174 *
175 * 1) Handle more ineligible cases.
176 *
177 * 2) Change ext4_fc_commit() to lookup logical to physical mapping using extent
178 * status tree. This would get rid of the need to call ext4_fc_track_inode()
179 * before acquiring i_data_sem. To do that we would need to ensure that
180 * modified extents from the extent status tree are not evicted from memory.
181 */
182
183#include <trace/events/ext4.h>
184static struct kmem_cache *ext4_fc_dentry_cachep;
185
186static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate)
187{
188 BUFFER_TRACE(bh, "");
189 if (uptodate) {
190 ext4_debug("%s: Block %lld up-to-date",
191 __func__, bh->b_blocknr);
192 set_buffer_uptodate(bh);
193 } else {
194 ext4_debug("%s: Block %lld not up-to-date",
195 __func__, bh->b_blocknr);
196 clear_buffer_uptodate(bh);
197 }
198
199 unlock_buffer(bh);
200}
201
202static inline void ext4_fc_reset_inode(struct inode *inode)
203{
204 struct ext4_inode_info *ei = EXT4_I(inode);
205
206 ei->i_fc_lblk_start = 0;
207 ei->i_fc_lblk_len = 0;
208}
209
210void ext4_fc_init_inode(struct inode *inode)
211{
212 struct ext4_inode_info *ei = EXT4_I(inode);
213
214 ext4_fc_reset_inode(inode);
215 ext4_clear_inode_state(inode, bit: EXT4_STATE_FC_COMMITTING);
216 INIT_LIST_HEAD(list: &ei->i_fc_list);
217 INIT_LIST_HEAD(list: &ei->i_fc_dilist);
218 init_waitqueue_head(&ei->i_fc_wait);
219}
220
221static bool ext4_fc_disabled(struct super_block *sb)
222{
223 return (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
224 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY));
225}
226
227/*
228 * Remove inode from fast commit list. If the inode is being committed
229 * we wait until inode commit is done.
230 */
231void ext4_fc_del(struct inode *inode)
232{
233 struct ext4_inode_info *ei = EXT4_I(inode);
234 struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
235 struct ext4_fc_dentry_update *fc_dentry;
236 wait_queue_head_t *wq;
237
238 if (ext4_fc_disabled(sb: inode->i_sb))
239 return;
240
241 mutex_lock(lock: &sbi->s_fc_lock);
242 if (list_empty(head: &ei->i_fc_list) && list_empty(head: &ei->i_fc_dilist)) {
243 mutex_unlock(lock: &sbi->s_fc_lock);
244 return;
245 }
246
247 /*
248 * Since ext4_fc_del is called from ext4_evict_inode while having a
249 * handle open, there is no need for us to wait here even if a fast
250 * commit is going on. That is because, if this inode is being
251 * committed, ext4_mark_inode_dirty would have waited for inode commit
252 * operation to finish before we come here. So, by the time we come
253 * here, inode's EXT4_STATE_FC_COMMITTING would have been cleared. So,
254 * we shouldn't see EXT4_STATE_FC_COMMITTING to be set on this inode
255 * here.
256 *
257 * We may come here without any handles open in the "no_delete" case of
258 * ext4_evict_inode as well. However, if that happens, we first mark the
259 * file system as fast commit ineligible anyway. So, even in that case,
260 * it is okay to remove the inode from the fc list.
261 */
262 WARN_ON(ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)
263 && !ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE));
264 while (ext4_test_inode_state(inode, bit: EXT4_STATE_FC_FLUSHING_DATA)) {
265#if (BITS_PER_LONG < 64)
266 DEFINE_WAIT_BIT(wait, &ei->i_state_flags,
267 EXT4_STATE_FC_FLUSHING_DATA);
268 wq = bit_waitqueue(&ei->i_state_flags,
269 EXT4_STATE_FC_FLUSHING_DATA);
270#else
271 DEFINE_WAIT_BIT(wait, &ei->i_flags,
272 EXT4_STATE_FC_FLUSHING_DATA);
273 wq = bit_waitqueue(word: &ei->i_flags,
274 bit: EXT4_STATE_FC_FLUSHING_DATA);
275#endif
276 prepare_to_wait(wq_head: wq, wq_entry: &wait.wq_entry, TASK_UNINTERRUPTIBLE);
277 if (ext4_test_inode_state(inode, bit: EXT4_STATE_FC_FLUSHING_DATA)) {
278 mutex_unlock(lock: &sbi->s_fc_lock);
279 schedule();
280 mutex_lock(lock: &sbi->s_fc_lock);
281 }
282 finish_wait(wq_head: wq, wq_entry: &wait.wq_entry);
283 }
284 list_del_init(entry: &ei->i_fc_list);
285
286 /*
287 * Since this inode is getting removed, let's also remove all FC
288 * dentry create references, since it is not needed to log it anyways.
289 */
290 if (list_empty(head: &ei->i_fc_dilist)) {
291 mutex_unlock(lock: &sbi->s_fc_lock);
292 return;
293 }
294
295 fc_dentry = list_first_entry(&ei->i_fc_dilist, struct ext4_fc_dentry_update, fcd_dilist);
296 WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT);
297 list_del_init(entry: &fc_dentry->fcd_list);
298 list_del_init(entry: &fc_dentry->fcd_dilist);
299
300 WARN_ON(!list_empty(&ei->i_fc_dilist));
301 mutex_unlock(lock: &sbi->s_fc_lock);
302
303 release_dentry_name_snapshot(&fc_dentry->fcd_name);
304 kmem_cache_free(s: ext4_fc_dentry_cachep, objp: fc_dentry);
305}
306
307/*
308 * Mark file system as fast commit ineligible, and record latest
309 * ineligible transaction tid. This means until the recorded
310 * transaction, commit operation would result in a full jbd2 commit.
311 */
312void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle)
313{
314 struct ext4_sb_info *sbi = EXT4_SB(sb);
315 tid_t tid;
316 bool has_transaction = true;
317 bool is_ineligible;
318
319 if (ext4_fc_disabled(sb))
320 return;
321
322 if (handle && !IS_ERR(ptr: handle))
323 tid = handle->h_transaction->t_tid;
324 else {
325 read_lock(&sbi->s_journal->j_state_lock);
326 if (sbi->s_journal->j_running_transaction)
327 tid = sbi->s_journal->j_running_transaction->t_tid;
328 else
329 has_transaction = false;
330 read_unlock(&sbi->s_journal->j_state_lock);
331 }
332 mutex_lock(lock: &sbi->s_fc_lock);
333 is_ineligible = ext4_test_mount_flag(sb, bit: EXT4_MF_FC_INELIGIBLE);
334 if (has_transaction && (!is_ineligible || tid_gt(x: tid, y: sbi->s_fc_ineligible_tid)))
335 sbi->s_fc_ineligible_tid = tid;
336 ext4_set_mount_flag(sb, bit: EXT4_MF_FC_INELIGIBLE);
337 mutex_unlock(lock: &sbi->s_fc_lock);
338 WARN_ON(reason >= EXT4_FC_REASON_MAX);
339 sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
340}
341
342/*
343 * Generic fast commit tracking function. If this is the first time this we are
344 * called after a full commit, we initialize fast commit fields and then call
345 * __fc_track_fn() with update = 0. If we have already been called after a full
346 * commit, we pass update = 1. Based on that, the track function can determine
347 * if it needs to track a field for the first time or if it needs to just
348 * update the previously tracked value.
349 *
350 * If enqueue is set, this function enqueues the inode in fast commit list.
351 */
352static int ext4_fc_track_template(
353 handle_t *handle, struct inode *inode,
354 int (*__fc_track_fn)(handle_t *handle, struct inode *, void *, bool),
355 void *args, int enqueue)
356{
357 bool update = false;
358 struct ext4_inode_info *ei = EXT4_I(inode);
359 struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
360 tid_t tid = 0;
361 int ret;
362
363 tid = handle->h_transaction->t_tid;
364 spin_lock(lock: &ei->i_fc_lock);
365 if (tid == ei->i_sync_tid) {
366 update = true;
367 } else {
368 ext4_fc_reset_inode(inode);
369 ei->i_sync_tid = tid;
370 }
371 ret = __fc_track_fn(handle, inode, args, update);
372 spin_unlock(lock: &ei->i_fc_lock);
373 if (!enqueue)
374 return ret;
375
376 mutex_lock(lock: &sbi->s_fc_lock);
377 if (list_empty(head: &EXT4_I(inode)->i_fc_list))
378 list_add_tail(new: &EXT4_I(inode)->i_fc_list,
379 head: (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
380 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ?
381 &sbi->s_fc_q[FC_Q_STAGING] :
382 &sbi->s_fc_q[FC_Q_MAIN]);
383 mutex_unlock(lock: &sbi->s_fc_lock);
384
385 return ret;
386}
387
388struct __track_dentry_update_args {
389 struct dentry *dentry;
390 int op;
391};
392
393/* __track_fn for directory entry updates. Called with ei->i_fc_lock. */
394static int __track_dentry_update(handle_t *handle, struct inode *inode,
395 void *arg, bool update)
396{
397 struct ext4_fc_dentry_update *node;
398 struct ext4_inode_info *ei = EXT4_I(inode);
399 struct __track_dentry_update_args *dentry_update =
400 (struct __track_dentry_update_args *)arg;
401 struct dentry *dentry = dentry_update->dentry;
402 struct inode *dir = dentry->d_parent->d_inode;
403 struct super_block *sb = inode->i_sb;
404 struct ext4_sb_info *sbi = EXT4_SB(sb);
405
406 spin_unlock(lock: &ei->i_fc_lock);
407
408 if (IS_ENCRYPTED(dir)) {
409 ext4_fc_mark_ineligible(sb, reason: EXT4_FC_REASON_ENCRYPTED_FILENAME,
410 handle);
411 spin_lock(lock: &ei->i_fc_lock);
412 return -EOPNOTSUPP;
413 }
414
415 node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);
416 if (!node) {
417 ext4_fc_mark_ineligible(sb, reason: EXT4_FC_REASON_NOMEM, handle);
418 spin_lock(lock: &ei->i_fc_lock);
419 return -ENOMEM;
420 }
421
422 node->fcd_op = dentry_update->op;
423 node->fcd_parent = dir->i_ino;
424 node->fcd_ino = inode->i_ino;
425 take_dentry_name_snapshot(&node->fcd_name, dentry);
426 INIT_LIST_HEAD(list: &node->fcd_dilist);
427 INIT_LIST_HEAD(list: &node->fcd_list);
428 mutex_lock(lock: &sbi->s_fc_lock);
429 if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
430 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING)
431 list_add_tail(new: &node->fcd_list,
432 head: &sbi->s_fc_dentry_q[FC_Q_STAGING]);
433 else
434 list_add_tail(new: &node->fcd_list, head: &sbi->s_fc_dentry_q[FC_Q_MAIN]);
435
436 /*
437 * This helps us keep a track of all fc_dentry updates which is part of
438 * this ext4 inode. So in case the inode is getting unlinked, before
439 * even we get a chance to fsync, we could remove all fc_dentry
440 * references while evicting the inode in ext4_fc_del().
441 * Also with this, we don't need to loop over all the inodes in
442 * sbi->s_fc_q to get the corresponding inode in
443 * ext4_fc_commit_dentry_updates().
444 */
445 if (dentry_update->op == EXT4_FC_TAG_CREAT) {
446 WARN_ON(!list_empty(&ei->i_fc_dilist));
447 list_add_tail(new: &node->fcd_dilist, head: &ei->i_fc_dilist);
448 }
449 mutex_unlock(lock: &sbi->s_fc_lock);
450 spin_lock(lock: &ei->i_fc_lock);
451
452 return 0;
453}
454
455void __ext4_fc_track_unlink(handle_t *handle,
456 struct inode *inode, struct dentry *dentry)
457{
458 struct __track_dentry_update_args args;
459 int ret;
460
461 args.dentry = dentry;
462 args.op = EXT4_FC_TAG_UNLINK;
463
464 ret = ext4_fc_track_template(handle, inode, fc_track_fn: __track_dentry_update,
465 args: (void *)&args, enqueue: 0);
466 trace_ext4_fc_track_unlink(handle, inode, dentry, ret);
467}
468
469void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry)
470{
471 struct inode *inode = d_inode(dentry);
472
473 if (ext4_fc_disabled(sb: inode->i_sb))
474 return;
475
476 if (ext4_test_mount_flag(sb: inode->i_sb, bit: EXT4_MF_FC_INELIGIBLE))
477 return;
478
479 __ext4_fc_track_unlink(handle, inode, dentry);
480}
481
482void __ext4_fc_track_link(handle_t *handle,
483 struct inode *inode, struct dentry *dentry)
484{
485 struct __track_dentry_update_args args;
486 int ret;
487
488 args.dentry = dentry;
489 args.op = EXT4_FC_TAG_LINK;
490
491 ret = ext4_fc_track_template(handle, inode, fc_track_fn: __track_dentry_update,
492 args: (void *)&args, enqueue: 0);
493 trace_ext4_fc_track_link(handle, inode, dentry, ret);
494}
495
496void ext4_fc_track_link(handle_t *handle, struct dentry *dentry)
497{
498 struct inode *inode = d_inode(dentry);
499
500 if (ext4_fc_disabled(sb: inode->i_sb))
501 return;
502
503 if (ext4_test_mount_flag(sb: inode->i_sb, bit: EXT4_MF_FC_INELIGIBLE))
504 return;
505
506 __ext4_fc_track_link(handle, inode, dentry);
507}
508
509void __ext4_fc_track_create(handle_t *handle, struct inode *inode,
510 struct dentry *dentry)
511{
512 struct __track_dentry_update_args args;
513 int ret;
514
515 args.dentry = dentry;
516 args.op = EXT4_FC_TAG_CREAT;
517
518 ret = ext4_fc_track_template(handle, inode, fc_track_fn: __track_dentry_update,
519 args: (void *)&args, enqueue: 0);
520 trace_ext4_fc_track_create(handle, inode, dentry, ret);
521}
522
523void ext4_fc_track_create(handle_t *handle, struct dentry *dentry)
524{
525 struct inode *inode = d_inode(dentry);
526
527 if (ext4_fc_disabled(sb: inode->i_sb))
528 return;
529
530 if (ext4_test_mount_flag(sb: inode->i_sb, bit: EXT4_MF_FC_INELIGIBLE))
531 return;
532
533 __ext4_fc_track_create(handle, inode, dentry);
534}
535
536/* __track_fn for inode tracking */
537static int __track_inode(handle_t *handle, struct inode *inode, void *arg,
538 bool update)
539{
540 if (update)
541 return -EEXIST;
542
543 EXT4_I(inode)->i_fc_lblk_len = 0;
544
545 return 0;
546}
547
548void ext4_fc_track_inode(handle_t *handle, struct inode *inode)
549{
550 struct ext4_inode_info *ei = EXT4_I(inode);
551 wait_queue_head_t *wq;
552 int ret;
553
554 if (S_ISDIR(inode->i_mode))
555 return;
556
557 if (ext4_fc_disabled(sb: inode->i_sb))
558 return;
559
560 if (ext4_should_journal_data(inode)) {
561 ext4_fc_mark_ineligible(sb: inode->i_sb,
562 reason: EXT4_FC_REASON_INODE_JOURNAL_DATA, handle);
563 return;
564 }
565
566 if (ext4_test_mount_flag(sb: inode->i_sb, bit: EXT4_MF_FC_INELIGIBLE))
567 return;
568
569 /*
570 * If we come here, we may sleep while waiting for the inode to
571 * commit. We shouldn't be holding i_data_sem when we go to sleep since
572 * the commit path needs to grab the lock while committing the inode.
573 */
574 lockdep_assert_not_held(&ei->i_data_sem);
575
576 while (ext4_test_inode_state(inode, bit: EXT4_STATE_FC_COMMITTING)) {
577#if (BITS_PER_LONG < 64)
578 DEFINE_WAIT_BIT(wait, &ei->i_state_flags,
579 EXT4_STATE_FC_COMMITTING);
580 wq = bit_waitqueue(&ei->i_state_flags,
581 EXT4_STATE_FC_COMMITTING);
582#else
583 DEFINE_WAIT_BIT(wait, &ei->i_flags,
584 EXT4_STATE_FC_COMMITTING);
585 wq = bit_waitqueue(word: &ei->i_flags,
586 bit: EXT4_STATE_FC_COMMITTING);
587#endif
588 prepare_to_wait(wq_head: wq, wq_entry: &wait.wq_entry, TASK_UNINTERRUPTIBLE);
589 if (ext4_test_inode_state(inode, bit: EXT4_STATE_FC_COMMITTING))
590 schedule();
591 finish_wait(wq_head: wq, wq_entry: &wait.wq_entry);
592 }
593
594 /*
595 * From this point on, this inode will not be committed either
596 * by fast or full commit as long as the handle is open.
597 */
598 ret = ext4_fc_track_template(handle, inode, fc_track_fn: __track_inode, NULL, enqueue: 1);
599 trace_ext4_fc_track_inode(handle, inode, ret);
600}
601
602struct __track_range_args {
603 ext4_lblk_t start, end;
604};
605
606/* __track_fn for tracking data updates */
607static int __track_range(handle_t *handle, struct inode *inode, void *arg,
608 bool update)
609{
610 struct ext4_inode_info *ei = EXT4_I(inode);
611 ext4_lblk_t oldstart;
612 struct __track_range_args *__arg =
613 (struct __track_range_args *)arg;
614
615 if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) {
616 ext4_debug("Special inode %ld being modified\n", inode->i_ino);
617 return -ECANCELED;
618 }
619
620 oldstart = ei->i_fc_lblk_start;
621
622 if (update && ei->i_fc_lblk_len > 0) {
623 ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start);
624 ei->i_fc_lblk_len =
625 max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) -
626 ei->i_fc_lblk_start + 1;
627 } else {
628 ei->i_fc_lblk_start = __arg->start;
629 ei->i_fc_lblk_len = __arg->end - __arg->start + 1;
630 }
631
632 return 0;
633}
634
635void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start,
636 ext4_lblk_t end)
637{
638 struct __track_range_args args;
639 int ret;
640
641 if (S_ISDIR(inode->i_mode))
642 return;
643
644 if (ext4_fc_disabled(sb: inode->i_sb))
645 return;
646
647 if (ext4_test_mount_flag(sb: inode->i_sb, bit: EXT4_MF_FC_INELIGIBLE))
648 return;
649
650 if (ext4_has_inline_data(inode)) {
651 ext4_fc_mark_ineligible(sb: inode->i_sb, reason: EXT4_FC_REASON_XATTR,
652 handle);
653 return;
654 }
655
656 args.start = start;
657 args.end = end;
658
659 ret = ext4_fc_track_template(handle, inode, fc_track_fn: __track_range, args: &args, enqueue: 1);
660
661 trace_ext4_fc_track_range(handle, inode, start, end, ret);
662}
663
664static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail)
665{
666 blk_opf_t write_flags = JBD2_JOURNAL_REQ_FLAGS;
667 struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;
668
669 /* Add REQ_FUA | REQ_PREFLUSH only its tail */
670 if (test_opt(sb, BARRIER) && is_tail)
671 write_flags |= REQ_FUA | REQ_PREFLUSH;
672 lock_buffer(bh);
673 set_buffer_dirty(bh);
674 set_buffer_uptodate(bh);
675 bh->b_end_io = ext4_end_buffer_io_sync;
676 submit_bh(REQ_OP_WRITE | write_flags, bh);
677 EXT4_SB(sb)->s_fc_bh = NULL;
678}
679
680/* Ext4 commit path routines */
681
682/*
683 * Allocate len bytes on a fast commit buffer.
684 *
685 * During the commit time this function is used to manage fast commit
686 * block space. We don't split a fast commit log onto different
687 * blocks. So this function makes sure that if there's not enough space
688 * on the current block, the remaining space in the current block is
689 * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case,
690 * new block is from jbd2 and CRC is updated to reflect the padding
691 * we added.
692 */
693static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)
694{
695 struct ext4_fc_tl tl;
696 struct ext4_sb_info *sbi = EXT4_SB(sb);
697 struct buffer_head *bh;
698 int bsize = sbi->s_journal->j_blocksize;
699 int ret, off = sbi->s_fc_bytes % bsize;
700 int remaining;
701 u8 *dst;
702
703 /*
704 * If 'len' is too long to fit in any block alongside a PAD tlv, then we
705 * cannot fulfill the request.
706 */
707 if (len > bsize - EXT4_FC_TAG_BASE_LEN)
708 return NULL;
709
710 if (!sbi->s_fc_bh) {
711 ret = jbd2_fc_get_buf(journal: EXT4_SB(sb)->s_journal, bh_out: &bh);
712 if (ret)
713 return NULL;
714 sbi->s_fc_bh = bh;
715 }
716 dst = sbi->s_fc_bh->b_data + off;
717
718 /*
719 * Allocate the bytes in the current block if we can do so while still
720 * leaving enough space for a PAD tlv.
721 */
722 remaining = bsize - EXT4_FC_TAG_BASE_LEN - off;
723 if (len <= remaining) {
724 sbi->s_fc_bytes += len;
725 return dst;
726 }
727
728 /*
729 * Else, terminate the current block with a PAD tlv, then allocate a new
730 * block and allocate the bytes at the start of that new block.
731 */
732
733 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD);
734 tl.fc_len = cpu_to_le16(remaining);
735 memcpy(to: dst, from: &tl, EXT4_FC_TAG_BASE_LEN);
736 memset(s: dst + EXT4_FC_TAG_BASE_LEN, c: 0, n: remaining);
737 *crc = ext4_chksum(crc: *crc, address: sbi->s_fc_bh->b_data, length: bsize);
738
739 ext4_fc_submit_bh(sb, is_tail: false);
740
741 ret = jbd2_fc_get_buf(journal: EXT4_SB(sb)->s_journal, bh_out: &bh);
742 if (ret)
743 return NULL;
744 sbi->s_fc_bh = bh;
745 sbi->s_fc_bytes += bsize - off + len;
746 return sbi->s_fc_bh->b_data;
747}
748
749/*
750 * Complete a fast commit by writing tail tag.
751 *
752 * Writing tail tag marks the end of a fast commit. In order to guarantee
753 * atomicity, after writing tail tag, even if there's space remaining
754 * in the block, next commit shouldn't use it. That's why tail tag
755 * has the length as that of the remaining space on the block.
756 */
757static int ext4_fc_write_tail(struct super_block *sb, u32 crc)
758{
759 struct ext4_sb_info *sbi = EXT4_SB(sb);
760 struct ext4_fc_tl tl;
761 struct ext4_fc_tail tail;
762 int off, bsize = sbi->s_journal->j_blocksize;
763 u8 *dst;
764
765 /*
766 * ext4_fc_reserve_space takes care of allocating an extra block if
767 * there's no enough space on this block for accommodating this tail.
768 */
769 dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), crc: &crc);
770 if (!dst)
771 return -ENOSPC;
772
773 off = sbi->s_fc_bytes % bsize;
774
775 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL);
776 tl.fc_len = cpu_to_le16(bsize - off + sizeof(struct ext4_fc_tail));
777 sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize);
778
779 memcpy(to: dst, from: &tl, EXT4_FC_TAG_BASE_LEN);
780 dst += EXT4_FC_TAG_BASE_LEN;
781 tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid);
782 memcpy(to: dst, from: &tail.fc_tid, len: sizeof(tail.fc_tid));
783 dst += sizeof(tail.fc_tid);
784 crc = ext4_chksum(crc, address: sbi->s_fc_bh->b_data,
785 length: dst - (u8 *)sbi->s_fc_bh->b_data);
786 tail.fc_crc = cpu_to_le32(crc);
787 memcpy(to: dst, from: &tail.fc_crc, len: sizeof(tail.fc_crc));
788 dst += sizeof(tail.fc_crc);
789 memset(s: dst, c: 0, n: bsize - off); /* Don't leak uninitialized memory. */
790
791 ext4_fc_submit_bh(sb, is_tail: true);
792
793 return 0;
794}
795
796/*
797 * Adds tag, length, value and updates CRC. Returns true if tlv was added.
798 * Returns false if there's not enough space.
799 */
800static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val,
801 u32 *crc)
802{
803 struct ext4_fc_tl tl;
804 u8 *dst;
805
806 dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc);
807 if (!dst)
808 return false;
809
810 tl.fc_tag = cpu_to_le16(tag);
811 tl.fc_len = cpu_to_le16(len);
812
813 memcpy(to: dst, from: &tl, EXT4_FC_TAG_BASE_LEN);
814 memcpy(to: dst + EXT4_FC_TAG_BASE_LEN, from: val, len);
815
816 return true;
817}
818
819/* Same as above, but adds dentry tlv. */
820static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc,
821 struct ext4_fc_dentry_update *fc_dentry)
822{
823 struct ext4_fc_dentry_info fcd;
824 struct ext4_fc_tl tl;
825 int dlen = fc_dentry->fcd_name.name.len;
826 u8 *dst = ext4_fc_reserve_space(sb,
827 EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc);
828
829 if (!dst)
830 return false;
831
832 fcd.fc_parent_ino = cpu_to_le32(fc_dentry->fcd_parent);
833 fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino);
834 tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op);
835 tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen);
836 memcpy(to: dst, from: &tl, EXT4_FC_TAG_BASE_LEN);
837 dst += EXT4_FC_TAG_BASE_LEN;
838 memcpy(to: dst, from: &fcd, len: sizeof(fcd));
839 dst += sizeof(fcd);
840 memcpy(to: dst, from: fc_dentry->fcd_name.name.name, len: dlen);
841
842 return true;
843}
844
845/*
846 * Writes inode in the fast commit space under TLV with tag @tag.
847 * Returns 0 on success, error on failure.
848 */
849static int ext4_fc_write_inode(struct inode *inode, u32 *crc)
850{
851 struct ext4_inode_info *ei = EXT4_I(inode);
852 int inode_len = EXT4_GOOD_OLD_INODE_SIZE;
853 int ret;
854 struct ext4_iloc iloc;
855 struct ext4_fc_inode fc_inode;
856 struct ext4_fc_tl tl;
857 u8 *dst;
858
859 ret = ext4_get_inode_loc(inode, &iloc);
860 if (ret)
861 return ret;
862
863 if (ext4_test_inode_flag(inode, bit: EXT4_INODE_INLINE_DATA))
864 inode_len = EXT4_INODE_SIZE(inode->i_sb);
865 else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE)
866 inode_len += ei->i_extra_isize;
867
868 fc_inode.fc_ino = cpu_to_le32(inode->i_ino);
869 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE);
870 tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino));
871
872 ret = -ECANCELED;
873 dst = ext4_fc_reserve_space(sb: inode->i_sb,
874 EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc);
875 if (!dst)
876 goto err;
877
878 memcpy(to: dst, from: &tl, EXT4_FC_TAG_BASE_LEN);
879 dst += EXT4_FC_TAG_BASE_LEN;
880 memcpy(to: dst, from: &fc_inode, len: sizeof(fc_inode));
881 dst += sizeof(fc_inode);
882 memcpy(to: dst, from: (u8 *)ext4_raw_inode(iloc: &iloc), len: inode_len);
883 ret = 0;
884err:
885 brelse(bh: iloc.bh);
886 return ret;
887}
888
889/*
890 * Writes updated data ranges for the inode in question. Updates CRC.
891 * Returns 0 on success, error otherwise.
892 */
893static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc)
894{
895 ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size;
896 struct ext4_inode_info *ei = EXT4_I(inode);
897 struct ext4_map_blocks map;
898 struct ext4_fc_add_range fc_ext;
899 struct ext4_fc_del_range lrange;
900 struct ext4_extent *ex;
901 int ret;
902
903 spin_lock(lock: &ei->i_fc_lock);
904 if (ei->i_fc_lblk_len == 0) {
905 spin_unlock(lock: &ei->i_fc_lock);
906 return 0;
907 }
908 old_blk_size = ei->i_fc_lblk_start;
909 new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1;
910 ei->i_fc_lblk_len = 0;
911 spin_unlock(lock: &ei->i_fc_lock);
912
913 cur_lblk_off = old_blk_size;
914 ext4_debug("will try writing %d to %d for inode %ld\n",
915 cur_lblk_off, new_blk_size, inode->i_ino);
916
917 while (cur_lblk_off <= new_blk_size) {
918 map.m_lblk = cur_lblk_off;
919 map.m_len = new_blk_size - cur_lblk_off + 1;
920 ret = ext4_map_blocks(NULL, inode, map: &map,
921 EXT4_GET_BLOCKS_IO_SUBMIT |
922 EXT4_EX_NOCACHE);
923 if (ret < 0)
924 return -ECANCELED;
925
926 if (map.m_len == 0) {
927 cur_lblk_off++;
928 continue;
929 }
930
931 if (ret == 0) {
932 lrange.fc_ino = cpu_to_le32(inode->i_ino);
933 lrange.fc_lblk = cpu_to_le32(map.m_lblk);
934 lrange.fc_len = cpu_to_le32(map.m_len);
935 if (!ext4_fc_add_tlv(sb: inode->i_sb, EXT4_FC_TAG_DEL_RANGE,
936 len: sizeof(lrange), val: (u8 *)&lrange, crc))
937 return -ENOSPC;
938 } else {
939 unsigned int max = (map.m_flags & EXT4_MAP_UNWRITTEN) ?
940 EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN;
941
942 /* Limit the number of blocks in one extent */
943 map.m_len = min(max, map.m_len);
944
945 fc_ext.fc_ino = cpu_to_le32(inode->i_ino);
946 ex = (struct ext4_extent *)&fc_ext.fc_ex;
947 ex->ee_block = cpu_to_le32(map.m_lblk);
948 ex->ee_len = cpu_to_le16(map.m_len);
949 ext4_ext_store_pblock(ex, pb: map.m_pblk);
950 if (map.m_flags & EXT4_MAP_UNWRITTEN)
951 ext4_ext_mark_unwritten(ext: ex);
952 else
953 ext4_ext_mark_initialized(ext: ex);
954 if (!ext4_fc_add_tlv(sb: inode->i_sb, EXT4_FC_TAG_ADD_RANGE,
955 len: sizeof(fc_ext), val: (u8 *)&fc_ext, crc))
956 return -ENOSPC;
957 }
958
959 cur_lblk_off += map.m_len;
960 }
961
962 return 0;
963}
964
965
966/* Flushes data of all the inodes in the commit queue. */
967static int ext4_fc_flush_data(journal_t *journal)
968{
969 struct super_block *sb = journal->j_private;
970 struct ext4_sb_info *sbi = EXT4_SB(sb);
971 struct ext4_inode_info *ei;
972 int ret = 0;
973
974 list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
975 ret = jbd2_submit_inode_data(journal, jinode: ei->jinode);
976 if (ret)
977 return ret;
978 }
979
980 list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
981 ret = jbd2_wait_inode_data(journal, jinode: ei->jinode);
982 if (ret)
983 return ret;
984 }
985
986 return 0;
987}
988
989/* Commit all the directory entry updates */
990static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc)
991{
992 struct super_block *sb = journal->j_private;
993 struct ext4_sb_info *sbi = EXT4_SB(sb);
994 struct ext4_fc_dentry_update *fc_dentry, *fc_dentry_n;
995 struct inode *inode;
996 struct ext4_inode_info *ei;
997 int ret;
998
999 if (list_empty(head: &sbi->s_fc_dentry_q[FC_Q_MAIN]))
1000 return 0;
1001 list_for_each_entry_safe(fc_dentry, fc_dentry_n,
1002 &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) {
1003 if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) {
1004 if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry))
1005 return -ENOSPC;
1006 continue;
1007 }
1008 /*
1009 * With fcd_dilist we need not loop in sbi->s_fc_q to get the
1010 * corresponding inode. Also, the corresponding inode could have been
1011 * deleted, in which case, we don't need to do anything.
1012 */
1013 if (list_empty(head: &fc_dentry->fcd_dilist))
1014 continue;
1015 ei = list_first_entry(&fc_dentry->fcd_dilist,
1016 struct ext4_inode_info, i_fc_dilist);
1017 inode = &ei->vfs_inode;
1018 WARN_ON(inode->i_ino != fc_dentry->fcd_ino);
1019
1020 /*
1021 * We first write the inode and then the create dirent. This
1022 * allows the recovery code to create an unnamed inode first
1023 * and then link it to a directory entry. This allows us
1024 * to use namei.c routines almost as is and simplifies
1025 * the recovery code.
1026 */
1027 ret = ext4_fc_write_inode(inode, crc);
1028 if (ret)
1029 return ret;
1030 ret = ext4_fc_write_inode_data(inode, crc);
1031 if (ret)
1032 return ret;
1033 if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry))
1034 return -ENOSPC;
1035 }
1036 return 0;
1037}
1038
1039static int ext4_fc_perform_commit(journal_t *journal)
1040{
1041 struct super_block *sb = journal->j_private;
1042 struct ext4_sb_info *sbi = EXT4_SB(sb);
1043 struct ext4_inode_info *iter;
1044 struct ext4_fc_head head;
1045 struct inode *inode;
1046 struct blk_plug plug;
1047 int ret = 0;
1048 u32 crc = 0;
1049
1050 /*
1051 * Step 1: Mark all inodes on s_fc_q[MAIN] with
1052 * EXT4_STATE_FC_FLUSHING_DATA. This prevents these inodes from being
1053 * freed until the data flush is over.
1054 */
1055 mutex_lock(lock: &sbi->s_fc_lock);
1056 list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
1057 ext4_set_inode_state(inode: &iter->vfs_inode,
1058 bit: EXT4_STATE_FC_FLUSHING_DATA);
1059 }
1060 mutex_unlock(lock: &sbi->s_fc_lock);
1061
1062 /* Step 2: Flush data for all the eligible inodes. */
1063 ret = ext4_fc_flush_data(journal);
1064
1065 /*
1066 * Step 3: Clear EXT4_STATE_FC_FLUSHING_DATA flag, before returning
1067 * any error from step 2. This ensures that waiters waiting on
1068 * EXT4_STATE_FC_FLUSHING_DATA can resume.
1069 */
1070 mutex_lock(lock: &sbi->s_fc_lock);
1071 list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
1072 ext4_clear_inode_state(inode: &iter->vfs_inode,
1073 bit: EXT4_STATE_FC_FLUSHING_DATA);
1074#if (BITS_PER_LONG < 64)
1075 wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_FLUSHING_DATA);
1076#else
1077 wake_up_bit(word: &iter->i_flags, bit: EXT4_STATE_FC_FLUSHING_DATA);
1078#endif
1079 }
1080
1081 /*
1082 * Make sure clearing of EXT4_STATE_FC_FLUSHING_DATA is visible before
1083 * the waiter checks the bit. Pairs with implicit barrier in
1084 * prepare_to_wait() in ext4_fc_del().
1085 */
1086 smp_mb();
1087 mutex_unlock(lock: &sbi->s_fc_lock);
1088
1089 /*
1090 * If we encountered error in Step 2, return it now after clearing
1091 * EXT4_STATE_FC_FLUSHING_DATA bit.
1092 */
1093 if (ret)
1094 return ret;
1095
1096
1097 /* Step 4: Mark all inodes as being committed. */
1098 jbd2_journal_lock_updates(journal);
1099 /*
1100 * The journal is now locked. No more handles can start and all the
1101 * previous handles are now drained. We now mark the inodes on the
1102 * commit queue as being committed.
1103 */
1104 mutex_lock(lock: &sbi->s_fc_lock);
1105 list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
1106 ext4_set_inode_state(inode: &iter->vfs_inode,
1107 bit: EXT4_STATE_FC_COMMITTING);
1108 }
1109 mutex_unlock(lock: &sbi->s_fc_lock);
1110 jbd2_journal_unlock_updates(journal);
1111
1112 /*
1113 * Step 5: If file system device is different from journal device,
1114 * issue a cache flush before we start writing fast commit blocks.
1115 */
1116 if (journal->j_fs_dev != journal->j_dev)
1117 blkdev_issue_flush(bdev: journal->j_fs_dev);
1118
1119 blk_start_plug(&plug);
1120 /* Step 6: Write fast commit blocks to disk. */
1121 if (sbi->s_fc_bytes == 0) {
1122 /*
1123 * Step 6.1: Add a head tag only if this is the first fast
1124 * commit in this TID.
1125 */
1126 head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES);
1127 head.fc_tid = cpu_to_le32(
1128 sbi->s_journal->j_running_transaction->t_tid);
1129 if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, len: sizeof(head),
1130 val: (u8 *)&head, crc: &crc)) {
1131 ret = -ENOSPC;
1132 goto out;
1133 }
1134 }
1135
1136 /* Step 6.2: Now write all the dentry updates. */
1137 mutex_lock(lock: &sbi->s_fc_lock);
1138 ret = ext4_fc_commit_dentry_updates(journal, crc: &crc);
1139 if (ret)
1140 goto out;
1141
1142 /* Step 6.3: Now write all the changed inodes to disk. */
1143 list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
1144 inode = &iter->vfs_inode;
1145 if (!ext4_test_inode_state(inode, bit: EXT4_STATE_FC_COMMITTING))
1146 continue;
1147
1148 ret = ext4_fc_write_inode_data(inode, crc: &crc);
1149 if (ret)
1150 goto out;
1151 ret = ext4_fc_write_inode(inode, crc: &crc);
1152 if (ret)
1153 goto out;
1154 }
1155 /* Step 6.4: Finally write tail tag to conclude this fast commit. */
1156 ret = ext4_fc_write_tail(sb, crc);
1157
1158out:
1159 mutex_unlock(lock: &sbi->s_fc_lock);
1160 blk_finish_plug(&plug);
1161 return ret;
1162}
1163
1164static void ext4_fc_update_stats(struct super_block *sb, int status,
1165 u64 commit_time, int nblks, tid_t commit_tid)
1166{
1167 struct ext4_fc_stats *stats = &EXT4_SB(sb)->s_fc_stats;
1168
1169 ext4_debug("Fast commit ended with status = %d for tid %u",
1170 status, commit_tid);
1171 if (status == EXT4_FC_STATUS_OK) {
1172 stats->fc_num_commits++;
1173 stats->fc_numblks += nblks;
1174 if (likely(stats->s_fc_avg_commit_time))
1175 stats->s_fc_avg_commit_time =
1176 (commit_time +
1177 stats->s_fc_avg_commit_time * 3) / 4;
1178 else
1179 stats->s_fc_avg_commit_time = commit_time;
1180 } else if (status == EXT4_FC_STATUS_FAILED ||
1181 status == EXT4_FC_STATUS_INELIGIBLE) {
1182 if (status == EXT4_FC_STATUS_FAILED)
1183 stats->fc_failed_commits++;
1184 stats->fc_ineligible_commits++;
1185 } else {
1186 stats->fc_skipped_commits++;
1187 }
1188 trace_ext4_fc_commit_stop(sb, nblks, reason: status, commit_tid);
1189}
1190
1191/*
1192 * The main commit entry point. Performs a fast commit for transaction
1193 * commit_tid if needed. If it's not possible to perform a fast commit
1194 * due to various reasons, we fall back to full commit. Returns 0
1195 * on success, error otherwise.
1196 */
1197int ext4_fc_commit(journal_t *journal, tid_t commit_tid)
1198{
1199 struct super_block *sb = journal->j_private;
1200 struct ext4_sb_info *sbi = EXT4_SB(sb);
1201 int nblks = 0, ret, bsize = journal->j_blocksize;
1202 int subtid = atomic_read(v: &sbi->s_fc_subtid);
1203 int status = EXT4_FC_STATUS_OK, fc_bufs_before = 0;
1204 ktime_t start_time, commit_time;
1205 int old_ioprio, journal_ioprio;
1206
1207 if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
1208 return jbd2_complete_transaction(journal, tid: commit_tid);
1209
1210 trace_ext4_fc_commit_start(sb, commit_tid);
1211
1212 start_time = ktime_get();
1213 old_ioprio = get_current_ioprio();
1214
1215restart_fc:
1216 ret = jbd2_fc_begin_commit(journal, tid: commit_tid);
1217 if (ret == -EALREADY) {
1218 /* There was an ongoing commit, check if we need to restart */
1219 if (atomic_read(v: &sbi->s_fc_subtid) <= subtid &&
1220 tid_gt(x: commit_tid, y: journal->j_commit_sequence))
1221 goto restart_fc;
1222 ext4_fc_update_stats(sb, status: EXT4_FC_STATUS_SKIPPED, commit_time: 0, nblks: 0,
1223 commit_tid);
1224 return 0;
1225 } else if (ret) {
1226 /*
1227 * Commit couldn't start. Just update stats and perform a
1228 * full commit.
1229 */
1230 ext4_fc_update_stats(sb, status: EXT4_FC_STATUS_FAILED, commit_time: 0, nblks: 0,
1231 commit_tid);
1232 return jbd2_complete_transaction(journal, tid: commit_tid);
1233 }
1234
1235 /*
1236 * After establishing journal barrier via jbd2_fc_begin_commit(), check
1237 * if we are fast commit ineligible.
1238 */
1239 if (ext4_test_mount_flag(sb, bit: EXT4_MF_FC_INELIGIBLE)) {
1240 status = EXT4_FC_STATUS_INELIGIBLE;
1241 goto fallback;
1242 }
1243
1244 /*
1245 * Now that we know that this thread is going to do a fast commit,
1246 * elevate the priority to match that of the journal thread.
1247 */
1248 if (journal->j_task->io_context)
1249 journal_ioprio = sbi->s_journal->j_task->io_context->ioprio;
1250 else
1251 journal_ioprio = EXT4_DEF_JOURNAL_IOPRIO;
1252 set_task_ioprio(current, ioprio: journal_ioprio);
1253 fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize;
1254 ret = ext4_fc_perform_commit(journal);
1255 if (ret < 0) {
1256 status = EXT4_FC_STATUS_FAILED;
1257 goto fallback;
1258 }
1259 nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before;
1260 ret = jbd2_fc_wait_bufs(journal, num_blks: nblks);
1261 if (ret < 0) {
1262 status = EXT4_FC_STATUS_FAILED;
1263 goto fallback;
1264 }
1265 atomic_inc(v: &sbi->s_fc_subtid);
1266 ret = jbd2_fc_end_commit(journal);
1267 set_task_ioprio(current, ioprio: old_ioprio);
1268 /*
1269 * weight the commit time higher than the average time so we
1270 * don't react too strongly to vast changes in the commit time
1271 */
1272 commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time));
1273 ext4_fc_update_stats(sb, status, commit_time, nblks, commit_tid);
1274 return ret;
1275
1276fallback:
1277 set_task_ioprio(current, ioprio: old_ioprio);
1278 ret = jbd2_fc_end_commit_fallback(journal);
1279 ext4_fc_update_stats(sb, status, commit_time: 0, nblks: 0, commit_tid);
1280 return ret;
1281}
1282
1283/*
1284 * Fast commit cleanup routine. This is called after every fast commit and
1285 * full commit. full is true if we are called after a full commit.
1286 */
1287static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid)
1288{
1289 struct super_block *sb = journal->j_private;
1290 struct ext4_sb_info *sbi = EXT4_SB(sb);
1291 struct ext4_inode_info *ei;
1292 struct ext4_fc_dentry_update *fc_dentry;
1293
1294 if (full && sbi->s_fc_bh)
1295 sbi->s_fc_bh = NULL;
1296
1297 trace_ext4_fc_cleanup(journal, full, tid);
1298 jbd2_fc_release_bufs(journal);
1299
1300 mutex_lock(lock: &sbi->s_fc_lock);
1301 while (!list_empty(head: &sbi->s_fc_q[FC_Q_MAIN])) {
1302 ei = list_first_entry(&sbi->s_fc_q[FC_Q_MAIN],
1303 struct ext4_inode_info,
1304 i_fc_list);
1305 list_del_init(entry: &ei->i_fc_list);
1306 ext4_clear_inode_state(inode: &ei->vfs_inode,
1307 bit: EXT4_STATE_FC_COMMITTING);
1308 if (tid_geq(x: tid, y: ei->i_sync_tid)) {
1309 ext4_fc_reset_inode(inode: &ei->vfs_inode);
1310 } else if (full) {
1311 /*
1312 * We are called after a full commit, inode has been
1313 * modified while the commit was running. Re-enqueue
1314 * the inode into STAGING, which will then be splice
1315 * back into MAIN. This cannot happen during
1316 * fastcommit because the journal is locked all the
1317 * time in that case (and tid doesn't increase so
1318 * tid check above isn't reliable).
1319 */
1320 list_add_tail(new: &ei->i_fc_list,
1321 head: &sbi->s_fc_q[FC_Q_STAGING]);
1322 }
1323 /*
1324 * Make sure clearing of EXT4_STATE_FC_COMMITTING is
1325 * visible before we send the wakeup. Pairs with implicit
1326 * barrier in prepare_to_wait() in ext4_fc_track_inode().
1327 */
1328 smp_mb();
1329#if (BITS_PER_LONG < 64)
1330 wake_up_bit(&ei->i_state_flags, EXT4_STATE_FC_COMMITTING);
1331#else
1332 wake_up_bit(word: &ei->i_flags, bit: EXT4_STATE_FC_COMMITTING);
1333#endif
1334 }
1335
1336 while (!list_empty(head: &sbi->s_fc_dentry_q[FC_Q_MAIN])) {
1337 fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN],
1338 struct ext4_fc_dentry_update,
1339 fcd_list);
1340 list_del_init(entry: &fc_dentry->fcd_list);
1341 list_del_init(entry: &fc_dentry->fcd_dilist);
1342
1343 release_dentry_name_snapshot(&fc_dentry->fcd_name);
1344 kmem_cache_free(s: ext4_fc_dentry_cachep, objp: fc_dentry);
1345 }
1346
1347 list_splice_init(list: &sbi->s_fc_dentry_q[FC_Q_STAGING],
1348 head: &sbi->s_fc_dentry_q[FC_Q_MAIN]);
1349 list_splice_init(list: &sbi->s_fc_q[FC_Q_STAGING],
1350 head: &sbi->s_fc_q[FC_Q_MAIN]);
1351
1352 if (tid_geq(x: tid, y: sbi->s_fc_ineligible_tid)) {
1353 sbi->s_fc_ineligible_tid = 0;
1354 ext4_clear_mount_flag(sb, bit: EXT4_MF_FC_INELIGIBLE);
1355 }
1356
1357 if (full)
1358 sbi->s_fc_bytes = 0;
1359 mutex_unlock(lock: &sbi->s_fc_lock);
1360 trace_ext4_fc_stats(sb);
1361}
1362
1363/* Ext4 Replay Path Routines */
1364
1365/* Helper struct for dentry replay routines */
1366struct dentry_info_args {
1367 int parent_ino, dname_len, ino, inode_len;
1368 char *dname;
1369};
1370
1371/* Same as struct ext4_fc_tl, but uses native endianness fields */
1372struct ext4_fc_tl_mem {
1373 u16 fc_tag;
1374 u16 fc_len;
1375};
1376
1377static inline void tl_to_darg(struct dentry_info_args *darg,
1378 struct ext4_fc_tl_mem *tl, u8 *val)
1379{
1380 struct ext4_fc_dentry_info fcd;
1381
1382 memcpy(to: &fcd, from: val, len: sizeof(fcd));
1383
1384 darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino);
1385 darg->ino = le32_to_cpu(fcd.fc_ino);
1386 darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname);
1387 darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info);
1388}
1389
1390static inline void ext4_fc_get_tl(struct ext4_fc_tl_mem *tl, u8 *val)
1391{
1392 struct ext4_fc_tl tl_disk;
1393
1394 memcpy(to: &tl_disk, from: val, EXT4_FC_TAG_BASE_LEN);
1395 tl->fc_len = le16_to_cpu(tl_disk.fc_len);
1396 tl->fc_tag = le16_to_cpu(tl_disk.fc_tag);
1397}
1398
1399/* Unlink replay function */
1400static int ext4_fc_replay_unlink(struct super_block *sb,
1401 struct ext4_fc_tl_mem *tl, u8 *val)
1402{
1403 struct inode *inode, *old_parent;
1404 struct qstr entry;
1405 struct dentry_info_args darg;
1406 int ret = 0;
1407
1408 tl_to_darg(darg: &darg, tl, val);
1409
1410 trace_ext4_fc_replay(sb, EXT4_FC_TAG_UNLINK, ino: darg.ino,
1411 priv1: darg.parent_ino, priv2: darg.dname_len);
1412
1413 entry.name = darg.dname;
1414 entry.len = darg.dname_len;
1415 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1416
1417 if (IS_ERR(ptr: inode)) {
1418 ext4_debug("Inode %d not found", darg.ino);
1419 return 0;
1420 }
1421
1422 old_parent = ext4_iget(sb, darg.parent_ino,
1423 EXT4_IGET_NORMAL);
1424 if (IS_ERR(ptr: old_parent)) {
1425 ext4_debug("Dir with inode %d not found", darg.parent_ino);
1426 iput(inode);
1427 return 0;
1428 }
1429
1430 ret = __ext4_unlink(dir: old_parent, d_name: &entry, inode, NULL);
1431 /* -ENOENT ok coz it might not exist anymore. */
1432 if (ret == -ENOENT)
1433 ret = 0;
1434 iput(old_parent);
1435 iput(inode);
1436 return ret;
1437}
1438
1439static int ext4_fc_replay_link_internal(struct super_block *sb,
1440 struct dentry_info_args *darg,
1441 struct inode *inode)
1442{
1443 struct inode *dir = NULL;
1444 struct dentry *dentry_dir = NULL, *dentry_inode = NULL;
1445 struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len);
1446 int ret = 0;
1447
1448 dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL);
1449 if (IS_ERR(ptr: dir)) {
1450 ext4_debug("Dir with inode %d not found.", darg->parent_ino);
1451 dir = NULL;
1452 goto out;
1453 }
1454
1455 dentry_dir = d_obtain_alias(dir);
1456 if (IS_ERR(ptr: dentry_dir)) {
1457 ext4_debug("Failed to obtain dentry");
1458 dentry_dir = NULL;
1459 goto out;
1460 }
1461
1462 dentry_inode = d_alloc(dentry_dir, &qstr_dname);
1463 if (!dentry_inode) {
1464 ext4_debug("Inode dentry not created.");
1465 ret = -ENOMEM;
1466 goto out;
1467 }
1468
1469 ret = __ext4_link(dir, inode, dentry: dentry_inode);
1470 /*
1471 * It's possible that link already existed since data blocks
1472 * for the dir in question got persisted before we crashed OR
1473 * we replayed this tag and crashed before the entire replay
1474 * could complete.
1475 */
1476 if (ret && ret != -EEXIST) {
1477 ext4_debug("Failed to link\n");
1478 goto out;
1479 }
1480
1481 ret = 0;
1482out:
1483 if (dentry_dir) {
1484 d_drop(dentry: dentry_dir);
1485 dput(dentry_dir);
1486 } else if (dir) {
1487 iput(dir);
1488 }
1489 if (dentry_inode) {
1490 d_drop(dentry: dentry_inode);
1491 dput(dentry_inode);
1492 }
1493
1494 return ret;
1495}
1496
1497/* Link replay function */
1498static int ext4_fc_replay_link(struct super_block *sb,
1499 struct ext4_fc_tl_mem *tl, u8 *val)
1500{
1501 struct inode *inode;
1502 struct dentry_info_args darg;
1503 int ret = 0;
1504
1505 tl_to_darg(darg: &darg, tl, val);
1506 trace_ext4_fc_replay(sb, EXT4_FC_TAG_LINK, ino: darg.ino,
1507 priv1: darg.parent_ino, priv2: darg.dname_len);
1508
1509 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1510 if (IS_ERR(ptr: inode)) {
1511 ext4_debug("Inode not found.");
1512 return 0;
1513 }
1514
1515 ret = ext4_fc_replay_link_internal(sb, darg: &darg, inode);
1516 iput(inode);
1517 return ret;
1518}
1519
1520/*
1521 * Record all the modified inodes during replay. We use this later to setup
1522 * block bitmaps correctly.
1523 */
1524static int ext4_fc_record_modified_inode(struct super_block *sb, int ino)
1525{
1526 struct ext4_fc_replay_state *state;
1527 int i;
1528
1529 state = &EXT4_SB(sb)->s_fc_replay_state;
1530 for (i = 0; i < state->fc_modified_inodes_used; i++)
1531 if (state->fc_modified_inodes[i] == ino)
1532 return 0;
1533 if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) {
1534 int *fc_modified_inodes;
1535
1536 fc_modified_inodes = krealloc(state->fc_modified_inodes,
1537 sizeof(int) * (state->fc_modified_inodes_size +
1538 EXT4_FC_REPLAY_REALLOC_INCREMENT),
1539 GFP_KERNEL);
1540 if (!fc_modified_inodes)
1541 return -ENOMEM;
1542 state->fc_modified_inodes = fc_modified_inodes;
1543 state->fc_modified_inodes_size +=
1544 EXT4_FC_REPLAY_REALLOC_INCREMENT;
1545 }
1546 state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino;
1547 return 0;
1548}
1549
1550/*
1551 * Inode replay function
1552 */
1553static int ext4_fc_replay_inode(struct super_block *sb,
1554 struct ext4_fc_tl_mem *tl, u8 *val)
1555{
1556 struct ext4_fc_inode fc_inode;
1557 struct ext4_inode *raw_inode;
1558 struct ext4_inode *raw_fc_inode;
1559 struct inode *inode = NULL;
1560 struct ext4_iloc iloc;
1561 int inode_len, ino, ret, tag = tl->fc_tag;
1562 struct ext4_extent_header *eh;
1563 size_t off_gen = offsetof(struct ext4_inode, i_generation);
1564
1565 memcpy(to: &fc_inode, from: val, len: sizeof(fc_inode));
1566
1567 ino = le32_to_cpu(fc_inode.fc_ino);
1568 trace_ext4_fc_replay(sb, tag, ino, priv1: 0, priv2: 0);
1569
1570 inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1571 if (!IS_ERR(ptr: inode)) {
1572 ext4_ext_clear_bb(inode);
1573 iput(inode);
1574 }
1575 inode = NULL;
1576
1577 ret = ext4_fc_record_modified_inode(sb, ino);
1578 if (ret)
1579 goto out;
1580
1581 raw_fc_inode = (struct ext4_inode *)
1582 (val + offsetof(struct ext4_fc_inode, fc_raw_inode));
1583 ret = ext4_get_fc_inode_loc(sb, ino, iloc: &iloc);
1584 if (ret)
1585 goto out;
1586
1587 inode_len = tl->fc_len - sizeof(struct ext4_fc_inode);
1588 raw_inode = ext4_raw_inode(iloc: &iloc);
1589
1590 memcpy(to: raw_inode, from: raw_fc_inode, offsetof(struct ext4_inode, i_block));
1591 memcpy(to: (u8 *)raw_inode + off_gen, from: (u8 *)raw_fc_inode + off_gen,
1592 len: inode_len - off_gen);
1593 if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) {
1594 eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]);
1595 if (eh->eh_magic != EXT4_EXT_MAGIC) {
1596 memset(s: eh, c: 0, n: sizeof(*eh));
1597 eh->eh_magic = EXT4_EXT_MAGIC;
1598 eh->eh_max = cpu_to_le16(
1599 (sizeof(raw_inode->i_block) -
1600 sizeof(struct ext4_extent_header))
1601 / sizeof(struct ext4_extent));
1602 }
1603 } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) {
1604 memcpy(to: raw_inode->i_block, from: raw_fc_inode->i_block,
1605 len: sizeof(raw_inode->i_block));
1606 }
1607
1608 /* Immediately update the inode on disk. */
1609 ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1610 if (ret)
1611 goto out;
1612 ret = sync_dirty_buffer(bh: iloc.bh);
1613 if (ret)
1614 goto out;
1615 ret = ext4_mark_inode_used(sb, ino);
1616 if (ret)
1617 goto out;
1618
1619 /* Given that we just wrote the inode on disk, this SHOULD succeed. */
1620 inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1621 if (IS_ERR(ptr: inode)) {
1622 ext4_debug("Inode not found.");
1623 return -EFSCORRUPTED;
1624 }
1625
1626 /*
1627 * Our allocator could have made different decisions than before
1628 * crashing. This should be fixed but until then, we calculate
1629 * the number of blocks the inode.
1630 */
1631 if (!ext4_test_inode_flag(inode, bit: EXT4_INODE_INLINE_DATA))
1632 ext4_ext_replay_set_iblocks(inode);
1633
1634 inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation);
1635 ext4_reset_inode_seed(inode);
1636
1637 ext4_inode_csum_set(inode, raw: ext4_raw_inode(iloc: &iloc), EXT4_I(inode));
1638 ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1639 sync_dirty_buffer(bh: iloc.bh);
1640 brelse(bh: iloc.bh);
1641out:
1642 iput(inode);
1643 if (!ret)
1644 blkdev_issue_flush(bdev: sb->s_bdev);
1645
1646 return 0;
1647}
1648
1649/*
1650 * Dentry create replay function.
1651 *
1652 * EXT4_FC_TAG_CREAT is preceded by EXT4_FC_TAG_INODE_FULL. Which means, the
1653 * inode for which we are trying to create a dentry here, should already have
1654 * been replayed before we start here.
1655 */
1656static int ext4_fc_replay_create(struct super_block *sb,
1657 struct ext4_fc_tl_mem *tl, u8 *val)
1658{
1659 int ret = 0;
1660 struct inode *inode = NULL;
1661 struct inode *dir = NULL;
1662 struct dentry_info_args darg;
1663
1664 tl_to_darg(darg: &darg, tl, val);
1665
1666 trace_ext4_fc_replay(sb, EXT4_FC_TAG_CREAT, ino: darg.ino,
1667 priv1: darg.parent_ino, priv2: darg.dname_len);
1668
1669 /* This takes care of update group descriptor and other metadata */
1670 ret = ext4_mark_inode_used(sb, ino: darg.ino);
1671 if (ret)
1672 goto out;
1673
1674 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1675 if (IS_ERR(ptr: inode)) {
1676 ext4_debug("inode %d not found.", darg.ino);
1677 inode = NULL;
1678 ret = -EINVAL;
1679 goto out;
1680 }
1681
1682 if (S_ISDIR(inode->i_mode)) {
1683 /*
1684 * If we are creating a directory, we need to make sure that the
1685 * dot and dot dot dirents are setup properly.
1686 */
1687 dir = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL);
1688 if (IS_ERR(ptr: dir)) {
1689 ext4_debug("Dir %d not found.", darg.ino);
1690 goto out;
1691 }
1692 ret = ext4_init_new_dir(NULL, dir, inode);
1693 iput(dir);
1694 if (ret) {
1695 ret = 0;
1696 goto out;
1697 }
1698 }
1699 ret = ext4_fc_replay_link_internal(sb, darg: &darg, inode);
1700 if (ret)
1701 goto out;
1702 set_nlink(inode, nlink: 1);
1703 ext4_mark_inode_dirty(NULL, inode);
1704out:
1705 iput(inode);
1706 return ret;
1707}
1708
1709/*
1710 * Record physical disk regions which are in use as per fast commit area,
1711 * and used by inodes during replay phase. Our simple replay phase
1712 * allocator excludes these regions from allocation.
1713 */
1714int ext4_fc_record_regions(struct super_block *sb, int ino,
1715 ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay)
1716{
1717 struct ext4_fc_replay_state *state;
1718 struct ext4_fc_alloc_region *region;
1719
1720 state = &EXT4_SB(sb)->s_fc_replay_state;
1721 /*
1722 * during replay phase, the fc_regions_valid may not same as
1723 * fc_regions_used, update it when do new additions.
1724 */
1725 if (replay && state->fc_regions_used != state->fc_regions_valid)
1726 state->fc_regions_used = state->fc_regions_valid;
1727 if (state->fc_regions_used == state->fc_regions_size) {
1728 struct ext4_fc_alloc_region *fc_regions;
1729
1730 fc_regions = krealloc(state->fc_regions,
1731 sizeof(struct ext4_fc_alloc_region) *
1732 (state->fc_regions_size +
1733 EXT4_FC_REPLAY_REALLOC_INCREMENT),
1734 GFP_KERNEL);
1735 if (!fc_regions)
1736 return -ENOMEM;
1737 state->fc_regions_size +=
1738 EXT4_FC_REPLAY_REALLOC_INCREMENT;
1739 state->fc_regions = fc_regions;
1740 }
1741 region = &state->fc_regions[state->fc_regions_used++];
1742 region->ino = ino;
1743 region->lblk = lblk;
1744 region->pblk = pblk;
1745 region->len = len;
1746
1747 if (replay)
1748 state->fc_regions_valid++;
1749
1750 return 0;
1751}
1752
1753/* Replay add range tag */
1754static int ext4_fc_replay_add_range(struct super_block *sb,
1755 struct ext4_fc_tl_mem *tl, u8 *val)
1756{
1757 struct ext4_fc_add_range fc_add_ex;
1758 struct ext4_extent newex, *ex;
1759 struct inode *inode;
1760 ext4_lblk_t start, cur;
1761 int remaining, len;
1762 ext4_fsblk_t start_pblk;
1763 struct ext4_map_blocks map;
1764 struct ext4_ext_path *path = NULL;
1765 int ret;
1766
1767 memcpy(to: &fc_add_ex, from: val, len: sizeof(fc_add_ex));
1768 ex = (struct ext4_extent *)&fc_add_ex.fc_ex;
1769
1770 trace_ext4_fc_replay(sb, EXT4_FC_TAG_ADD_RANGE,
1771 le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block),
1772 priv2: ext4_ext_get_actual_len(ext: ex));
1773
1774 inode = ext4_iget(sb, le32_to_cpu(fc_add_ex.fc_ino), EXT4_IGET_NORMAL);
1775 if (IS_ERR(ptr: inode)) {
1776 ext4_debug("Inode not found.");
1777 return 0;
1778 }
1779
1780 ret = ext4_fc_record_modified_inode(sb, ino: inode->i_ino);
1781 if (ret)
1782 goto out;
1783
1784 start = le32_to_cpu(ex->ee_block);
1785 start_pblk = ext4_ext_pblock(ex);
1786 len = ext4_ext_get_actual_len(ext: ex);
1787
1788 cur = start;
1789 remaining = len;
1790 ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %ld\n",
1791 start, start_pblk, len, ext4_ext_is_unwritten(ex),
1792 inode->i_ino);
1793
1794 while (remaining > 0) {
1795 map.m_lblk = cur;
1796 map.m_len = remaining;
1797 map.m_pblk = 0;
1798 ret = ext4_map_blocks(NULL, inode, map: &map, flags: 0);
1799
1800 if (ret < 0)
1801 goto out;
1802
1803 if (ret == 0) {
1804 /* Range is not mapped */
1805 path = ext4_find_extent(inode, cur, path, flags: 0);
1806 if (IS_ERR(ptr: path))
1807 goto out;
1808 memset(s: &newex, c: 0, n: sizeof(newex));
1809 newex.ee_block = cpu_to_le32(cur);
1810 ext4_ext_store_pblock(
1811 ex: &newex, pb: start_pblk + cur - start);
1812 newex.ee_len = cpu_to_le16(map.m_len);
1813 if (ext4_ext_is_unwritten(ext: ex))
1814 ext4_ext_mark_unwritten(ext: &newex);
1815 down_write(sem: &EXT4_I(inode)->i_data_sem);
1816 path = ext4_ext_insert_extent(NULL, inode,
1817 path, newext: &newex, gb_flags: 0);
1818 up_write(sem: (&EXT4_I(inode)->i_data_sem));
1819 if (IS_ERR(ptr: path))
1820 goto out;
1821 goto next;
1822 }
1823
1824 if (start_pblk + cur - start != map.m_pblk) {
1825 /*
1826 * Logical to physical mapping changed. This can happen
1827 * if this range was removed and then reallocated to
1828 * map to new physical blocks during a fast commit.
1829 */
1830 ret = ext4_ext_replay_update_ex(inode, start: cur, len: map.m_len,
1831 unwritten: ext4_ext_is_unwritten(ext: ex),
1832 pblk: start_pblk + cur - start);
1833 if (ret)
1834 goto out;
1835 /*
1836 * Mark the old blocks as free since they aren't used
1837 * anymore. We maintain an array of all the modified
1838 * inodes. In case these blocks are still used at either
1839 * a different logical range in the same inode or in
1840 * some different inode, we will mark them as allocated
1841 * at the end of the FC replay using our array of
1842 * modified inodes.
1843 */
1844 ext4_mb_mark_bb(sb: inode->i_sb, block: map.m_pblk, len: map.m_len, state: false);
1845 goto next;
1846 }
1847
1848 /* Range is mapped and needs a state change */
1849 ext4_debug("Converting from %ld to %d %lld",
1850 map.m_flags & EXT4_MAP_UNWRITTEN,
1851 ext4_ext_is_unwritten(ex), map.m_pblk);
1852 ret = ext4_ext_replay_update_ex(inode, start: cur, len: map.m_len,
1853 unwritten: ext4_ext_is_unwritten(ext: ex), pblk: map.m_pblk);
1854 if (ret)
1855 goto out;
1856 /*
1857 * We may have split the extent tree while toggling the state.
1858 * Try to shrink the extent tree now.
1859 */
1860 ext4_ext_replay_shrink_inode(inode, end: start + len);
1861next:
1862 cur += map.m_len;
1863 remaining -= map.m_len;
1864 }
1865 ext4_ext_replay_shrink_inode(inode, end: i_size_read(inode) >>
1866 sb->s_blocksize_bits);
1867out:
1868 ext4_free_ext_path(path);
1869 iput(inode);
1870 return 0;
1871}
1872
1873/* Replay DEL_RANGE tag */
1874static int
1875ext4_fc_replay_del_range(struct super_block *sb,
1876 struct ext4_fc_tl_mem *tl, u8 *val)
1877{
1878 struct inode *inode;
1879 struct ext4_fc_del_range lrange;
1880 struct ext4_map_blocks map;
1881 ext4_lblk_t cur, remaining;
1882 int ret;
1883
1884 memcpy(to: &lrange, from: val, len: sizeof(lrange));
1885 cur = le32_to_cpu(lrange.fc_lblk);
1886 remaining = le32_to_cpu(lrange.fc_len);
1887
1888 trace_ext4_fc_replay(sb, EXT4_FC_TAG_DEL_RANGE,
1889 le32_to_cpu(lrange.fc_ino), priv1: cur, priv2: remaining);
1890
1891 inode = ext4_iget(sb, le32_to_cpu(lrange.fc_ino), EXT4_IGET_NORMAL);
1892 if (IS_ERR(ptr: inode)) {
1893 ext4_debug("Inode %d not found", le32_to_cpu(lrange.fc_ino));
1894 return 0;
1895 }
1896
1897 ret = ext4_fc_record_modified_inode(sb, ino: inode->i_ino);
1898 if (ret)
1899 goto out;
1900
1901 ext4_debug("DEL_RANGE, inode %ld, lblk %d, len %d\n",
1902 inode->i_ino, le32_to_cpu(lrange.fc_lblk),
1903 le32_to_cpu(lrange.fc_len));
1904 while (remaining > 0) {
1905 map.m_lblk = cur;
1906 map.m_len = remaining;
1907
1908 ret = ext4_map_blocks(NULL, inode, map: &map, flags: 0);
1909 if (ret < 0)
1910 goto out;
1911 if (ret > 0) {
1912 remaining -= ret;
1913 cur += ret;
1914 ext4_mb_mark_bb(sb: inode->i_sb, block: map.m_pblk, len: map.m_len, state: false);
1915 } else {
1916 remaining -= map.m_len;
1917 cur += map.m_len;
1918 }
1919 }
1920
1921 down_write(sem: &EXT4_I(inode)->i_data_sem);
1922 ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk),
1923 le32_to_cpu(lrange.fc_lblk) +
1924 le32_to_cpu(lrange.fc_len) - 1);
1925 up_write(sem: &EXT4_I(inode)->i_data_sem);
1926 if (ret)
1927 goto out;
1928 ext4_ext_replay_shrink_inode(inode,
1929 end: i_size_read(inode) >> sb->s_blocksize_bits);
1930 ext4_mark_inode_dirty(NULL, inode);
1931out:
1932 iput(inode);
1933 return 0;
1934}
1935
1936static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb)
1937{
1938 struct ext4_fc_replay_state *state;
1939 struct inode *inode;
1940 struct ext4_ext_path *path = NULL;
1941 struct ext4_map_blocks map;
1942 int i, ret, j;
1943 ext4_lblk_t cur, end;
1944
1945 state = &EXT4_SB(sb)->s_fc_replay_state;
1946 for (i = 0; i < state->fc_modified_inodes_used; i++) {
1947 inode = ext4_iget(sb, state->fc_modified_inodes[i],
1948 EXT4_IGET_NORMAL);
1949 if (IS_ERR(ptr: inode)) {
1950 ext4_debug("Inode %d not found.",
1951 state->fc_modified_inodes[i]);
1952 continue;
1953 }
1954 cur = 0;
1955 end = EXT_MAX_BLOCKS;
1956 if (ext4_test_inode_flag(inode, bit: EXT4_INODE_INLINE_DATA)) {
1957 iput(inode);
1958 continue;
1959 }
1960 while (cur < end) {
1961 map.m_lblk = cur;
1962 map.m_len = end - cur;
1963
1964 ret = ext4_map_blocks(NULL, inode, map: &map, flags: 0);
1965 if (ret < 0)
1966 break;
1967
1968 if (ret > 0) {
1969 path = ext4_find_extent(inode, map.m_lblk, path, flags: 0);
1970 if (!IS_ERR(ptr: path)) {
1971 for (j = 0; j < path->p_depth; j++)
1972 ext4_mb_mark_bb(sb: inode->i_sb,
1973 block: path[j].p_block, len: 1, state: true);
1974 } else {
1975 path = NULL;
1976 }
1977 cur += ret;
1978 ext4_mb_mark_bb(sb: inode->i_sb, block: map.m_pblk,
1979 len: map.m_len, state: true);
1980 } else {
1981 cur = cur + (map.m_len ? map.m_len : 1);
1982 }
1983 }
1984 iput(inode);
1985 }
1986
1987 ext4_free_ext_path(path);
1988}
1989
1990/*
1991 * Check if block is in excluded regions for block allocation. The simple
1992 * allocator that runs during replay phase is calls this function to see
1993 * if it is okay to use a block.
1994 */
1995bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t blk)
1996{
1997 int i;
1998 struct ext4_fc_replay_state *state;
1999
2000 state = &EXT4_SB(sb)->s_fc_replay_state;
2001 for (i = 0; i < state->fc_regions_valid; i++) {
2002 if (state->fc_regions[i].ino == 0 ||
2003 state->fc_regions[i].len == 0)
2004 continue;
2005 if (in_range(blk, state->fc_regions[i].pblk,
2006 state->fc_regions[i].len))
2007 return true;
2008 }
2009 return false;
2010}
2011
2012/* Cleanup function called after replay */
2013void ext4_fc_replay_cleanup(struct super_block *sb)
2014{
2015 struct ext4_sb_info *sbi = EXT4_SB(sb);
2016
2017 sbi->s_mount_state &= ~EXT4_FC_REPLAY;
2018 kfree(objp: sbi->s_fc_replay_state.fc_regions);
2019 kfree(objp: sbi->s_fc_replay_state.fc_modified_inodes);
2020}
2021
2022static bool ext4_fc_value_len_isvalid(struct ext4_sb_info *sbi,
2023 int tag, int len)
2024{
2025 switch (tag) {
2026 case EXT4_FC_TAG_ADD_RANGE:
2027 return len == sizeof(struct ext4_fc_add_range);
2028 case EXT4_FC_TAG_DEL_RANGE:
2029 return len == sizeof(struct ext4_fc_del_range);
2030 case EXT4_FC_TAG_CREAT:
2031 case EXT4_FC_TAG_LINK:
2032 case EXT4_FC_TAG_UNLINK:
2033 len -= sizeof(struct ext4_fc_dentry_info);
2034 return len >= 1 && len <= EXT4_NAME_LEN;
2035 case EXT4_FC_TAG_INODE:
2036 len -= sizeof(struct ext4_fc_inode);
2037 return len >= EXT4_GOOD_OLD_INODE_SIZE &&
2038 len <= sbi->s_inode_size;
2039 case EXT4_FC_TAG_PAD:
2040 return true; /* padding can have any length */
2041 case EXT4_FC_TAG_TAIL:
2042 return len >= sizeof(struct ext4_fc_tail);
2043 case EXT4_FC_TAG_HEAD:
2044 return len == sizeof(struct ext4_fc_head);
2045 }
2046 return false;
2047}
2048
2049/*
2050 * Recovery Scan phase handler
2051 *
2052 * This function is called during the scan phase and is responsible
2053 * for doing following things:
2054 * - Make sure the fast commit area has valid tags for replay
2055 * - Count number of tags that need to be replayed by the replay handler
2056 * - Verify CRC
2057 * - Create a list of excluded blocks for allocation during replay phase
2058 *
2059 * This function returns JBD2_FC_REPLAY_CONTINUE to indicate that SCAN is
2060 * incomplete and JBD2 should send more blocks. It returns JBD2_FC_REPLAY_STOP
2061 * to indicate that scan has finished and JBD2 can now start replay phase.
2062 * It returns a negative error to indicate that there was an error. At the end
2063 * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set
2064 * to indicate the number of tags that need to replayed during the replay phase.
2065 */
2066static int ext4_fc_replay_scan(journal_t *journal,
2067 struct buffer_head *bh, int off,
2068 tid_t expected_tid)
2069{
2070 struct super_block *sb = journal->j_private;
2071 struct ext4_sb_info *sbi = EXT4_SB(sb);
2072 struct ext4_fc_replay_state *state;
2073 int ret = JBD2_FC_REPLAY_CONTINUE;
2074 struct ext4_fc_add_range ext;
2075 struct ext4_fc_tl_mem tl;
2076 struct ext4_fc_tail tail;
2077 __u8 *start, *end, *cur, *val;
2078 struct ext4_fc_head head;
2079 struct ext4_extent *ex;
2080
2081 state = &sbi->s_fc_replay_state;
2082
2083 start = (u8 *)bh->b_data;
2084 end = start + journal->j_blocksize;
2085
2086 if (state->fc_replay_expected_off == 0) {
2087 state->fc_cur_tag = 0;
2088 state->fc_replay_num_tags = 0;
2089 state->fc_crc = 0;
2090 state->fc_regions = NULL;
2091 state->fc_regions_valid = state->fc_regions_used =
2092 state->fc_regions_size = 0;
2093 /* Check if we can stop early */
2094 if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag)
2095 != EXT4_FC_TAG_HEAD)
2096 return 0;
2097 }
2098
2099 if (off != state->fc_replay_expected_off) {
2100 ret = -EFSCORRUPTED;
2101 goto out_err;
2102 }
2103
2104 state->fc_replay_expected_off++;
2105 for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN;
2106 cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) {
2107 ext4_fc_get_tl(tl: &tl, val: cur);
2108 val = cur + EXT4_FC_TAG_BASE_LEN;
2109 if (tl.fc_len > end - val ||
2110 !ext4_fc_value_len_isvalid(sbi, tag: tl.fc_tag, len: tl.fc_len)) {
2111 ret = state->fc_replay_num_tags ?
2112 JBD2_FC_REPLAY_STOP : -ECANCELED;
2113 goto out_err;
2114 }
2115 ext4_debug("Scan phase, tag:%s, blk %lld\n",
2116 tag2str(tl.fc_tag), bh->b_blocknr);
2117 switch (tl.fc_tag) {
2118 case EXT4_FC_TAG_ADD_RANGE:
2119 memcpy(to: &ext, from: val, len: sizeof(ext));
2120 ex = (struct ext4_extent *)&ext.fc_ex;
2121 ret = ext4_fc_record_regions(sb,
2122 le32_to_cpu(ext.fc_ino),
2123 le32_to_cpu(ex->ee_block), pblk: ext4_ext_pblock(ex),
2124 len: ext4_ext_get_actual_len(ext: ex), replay: 0);
2125 if (ret < 0)
2126 break;
2127 ret = JBD2_FC_REPLAY_CONTINUE;
2128 fallthrough;
2129 case EXT4_FC_TAG_DEL_RANGE:
2130 case EXT4_FC_TAG_LINK:
2131 case EXT4_FC_TAG_UNLINK:
2132 case EXT4_FC_TAG_CREAT:
2133 case EXT4_FC_TAG_INODE:
2134 case EXT4_FC_TAG_PAD:
2135 state->fc_cur_tag++;
2136 state->fc_crc = ext4_chksum(crc: state->fc_crc, address: cur,
2137 EXT4_FC_TAG_BASE_LEN + tl.fc_len);
2138 break;
2139 case EXT4_FC_TAG_TAIL:
2140 state->fc_cur_tag++;
2141 memcpy(to: &tail, from: val, len: sizeof(tail));
2142 state->fc_crc = ext4_chksum(crc: state->fc_crc, address: cur,
2143 EXT4_FC_TAG_BASE_LEN +
2144 offsetof(struct ext4_fc_tail,
2145 fc_crc));
2146 if (le32_to_cpu(tail.fc_tid) == expected_tid &&
2147 le32_to_cpu(tail.fc_crc) == state->fc_crc) {
2148 state->fc_replay_num_tags = state->fc_cur_tag;
2149 state->fc_regions_valid =
2150 state->fc_regions_used;
2151 } else {
2152 ret = state->fc_replay_num_tags ?
2153 JBD2_FC_REPLAY_STOP : -EFSBADCRC;
2154 }
2155 state->fc_crc = 0;
2156 break;
2157 case EXT4_FC_TAG_HEAD:
2158 memcpy(to: &head, from: val, len: sizeof(head));
2159 if (le32_to_cpu(head.fc_features) &
2160 ~EXT4_FC_SUPPORTED_FEATURES) {
2161 ret = -EOPNOTSUPP;
2162 break;
2163 }
2164 if (le32_to_cpu(head.fc_tid) != expected_tid) {
2165 ret = JBD2_FC_REPLAY_STOP;
2166 break;
2167 }
2168 state->fc_cur_tag++;
2169 state->fc_crc = ext4_chksum(crc: state->fc_crc, address: cur,
2170 EXT4_FC_TAG_BASE_LEN + tl.fc_len);
2171 break;
2172 default:
2173 ret = state->fc_replay_num_tags ?
2174 JBD2_FC_REPLAY_STOP : -ECANCELED;
2175 }
2176 if (ret < 0 || ret == JBD2_FC_REPLAY_STOP)
2177 break;
2178 }
2179
2180out_err:
2181 trace_ext4_fc_replay_scan(sb, error: ret, off);
2182 return ret;
2183}
2184
2185/*
2186 * Main recovery path entry point.
2187 * The meaning of return codes is similar as above.
2188 */
2189static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh,
2190 enum passtype pass, int off, tid_t expected_tid)
2191{
2192 struct super_block *sb = journal->j_private;
2193 struct ext4_sb_info *sbi = EXT4_SB(sb);
2194 struct ext4_fc_tl_mem tl;
2195 __u8 *start, *end, *cur, *val;
2196 int ret = JBD2_FC_REPLAY_CONTINUE;
2197 struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state;
2198 struct ext4_fc_tail tail;
2199
2200 if (pass == PASS_SCAN) {
2201 state->fc_current_pass = PASS_SCAN;
2202 return ext4_fc_replay_scan(journal, bh, off, expected_tid);
2203 }
2204
2205 if (state->fc_current_pass != pass) {
2206 state->fc_current_pass = pass;
2207 sbi->s_mount_state |= EXT4_FC_REPLAY;
2208 }
2209 if (!sbi->s_fc_replay_state.fc_replay_num_tags) {
2210 ext4_debug("Replay stops\n");
2211 ext4_fc_set_bitmaps_and_counters(sb);
2212 return 0;
2213 }
2214
2215#ifdef CONFIG_EXT4_DEBUG
2216 if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) {
2217 pr_warn("Dropping fc block %d because max_replay set\n", off);
2218 return JBD2_FC_REPLAY_STOP;
2219 }
2220#endif
2221
2222 start = (u8 *)bh->b_data;
2223 end = start + journal->j_blocksize;
2224
2225 for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN;
2226 cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) {
2227 ext4_fc_get_tl(tl: &tl, val: cur);
2228 val = cur + EXT4_FC_TAG_BASE_LEN;
2229
2230 if (state->fc_replay_num_tags == 0) {
2231 ret = JBD2_FC_REPLAY_STOP;
2232 ext4_fc_set_bitmaps_and_counters(sb);
2233 break;
2234 }
2235
2236 ext4_debug("Replay phase, tag:%s\n", tag2str(tl.fc_tag));
2237 state->fc_replay_num_tags--;
2238 switch (tl.fc_tag) {
2239 case EXT4_FC_TAG_LINK:
2240 ret = ext4_fc_replay_link(sb, tl: &tl, val);
2241 break;
2242 case EXT4_FC_TAG_UNLINK:
2243 ret = ext4_fc_replay_unlink(sb, tl: &tl, val);
2244 break;
2245 case EXT4_FC_TAG_ADD_RANGE:
2246 ret = ext4_fc_replay_add_range(sb, tl: &tl, val);
2247 break;
2248 case EXT4_FC_TAG_CREAT:
2249 ret = ext4_fc_replay_create(sb, tl: &tl, val);
2250 break;
2251 case EXT4_FC_TAG_DEL_RANGE:
2252 ret = ext4_fc_replay_del_range(sb, tl: &tl, val);
2253 break;
2254 case EXT4_FC_TAG_INODE:
2255 ret = ext4_fc_replay_inode(sb, tl: &tl, val);
2256 break;
2257 case EXT4_FC_TAG_PAD:
2258 trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, ino: 0,
2259 priv1: tl.fc_len, priv2: 0);
2260 break;
2261 case EXT4_FC_TAG_TAIL:
2262 trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL,
2263 ino: 0, priv1: tl.fc_len, priv2: 0);
2264 memcpy(to: &tail, from: val, len: sizeof(tail));
2265 WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid);
2266 break;
2267 case EXT4_FC_TAG_HEAD:
2268 break;
2269 default:
2270 trace_ext4_fc_replay(sb, tag: tl.fc_tag, ino: 0, priv1: tl.fc_len, priv2: 0);
2271 ret = -ECANCELED;
2272 break;
2273 }
2274 if (ret < 0)
2275 break;
2276 ret = JBD2_FC_REPLAY_CONTINUE;
2277 }
2278 return ret;
2279}
2280
2281void ext4_fc_init(struct super_block *sb, journal_t *journal)
2282{
2283 /*
2284 * We set replay callback even if fast commit disabled because we may
2285 * could still have fast commit blocks that need to be replayed even if
2286 * fast commit has now been turned off.
2287 */
2288 journal->j_fc_replay_callback = ext4_fc_replay;
2289 if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
2290 return;
2291 journal->j_fc_cleanup_callback = ext4_fc_cleanup;
2292}
2293
2294static const char * const fc_ineligible_reasons[] = {
2295 [EXT4_FC_REASON_XATTR] = "Extended attributes changed",
2296 [EXT4_FC_REASON_CROSS_RENAME] = "Cross rename",
2297 [EXT4_FC_REASON_JOURNAL_FLAG_CHANGE] = "Journal flag changed",
2298 [EXT4_FC_REASON_NOMEM] = "Insufficient memory",
2299 [EXT4_FC_REASON_SWAP_BOOT] = "Swap boot",
2300 [EXT4_FC_REASON_RESIZE] = "Resize",
2301 [EXT4_FC_REASON_RENAME_DIR] = "Dir renamed",
2302 [EXT4_FC_REASON_FALLOC_RANGE] = "Falloc range op",
2303 [EXT4_FC_REASON_INODE_JOURNAL_DATA] = "Data journalling",
2304 [EXT4_FC_REASON_ENCRYPTED_FILENAME] = "Encrypted filename",
2305};
2306
2307int ext4_fc_info_show(struct seq_file *seq, void *v)
2308{
2309 struct ext4_sb_info *sbi = EXT4_SB(sb: (struct super_block *)seq->private);
2310 struct ext4_fc_stats *stats = &sbi->s_fc_stats;
2311 int i;
2312
2313 if (v != SEQ_START_TOKEN)
2314 return 0;
2315
2316 seq_printf(m: seq,
2317 fmt: "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n",
2318 stats->fc_num_commits, stats->fc_ineligible_commits,
2319 stats->fc_numblks,
2320 div_u64(dividend: stats->s_fc_avg_commit_time, divisor: 1000));
2321 seq_puts(m: seq, s: "Ineligible reasons:\n");
2322 for (i = 0; i < EXT4_FC_REASON_MAX; i++)
2323 seq_printf(m: seq, fmt: "\"%s\":\t%d\n", fc_ineligible_reasons[i],
2324 stats->fc_ineligible_reason_count[i]);
2325
2326 return 0;
2327}
2328
2329int __init ext4_fc_init_dentry_cache(void)
2330{
2331 ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update,
2332 SLAB_RECLAIM_ACCOUNT);
2333
2334 if (ext4_fc_dentry_cachep == NULL)
2335 return -ENOMEM;
2336
2337 return 0;
2338}
2339
2340void ext4_fc_destroy_dentry_cache(void)
2341{
2342 kmem_cache_destroy(s: ext4_fc_dentry_cachep);
2343}
2344