blob: 44b192fdfb59f49b780300fe20898b1d57c7c826 [file] [log] [blame]
Harshad Shirwadkar6866d7b2020-10-15 13:37:55 -07001// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * fs/ext4/fast_commit.c
5 *
6 * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com>
7 *
8 * Ext4 fast commits routines.
9 */
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -070010#include "ext4.h"
Harshad Shirwadkar6866d7b2020-10-15 13:37:55 -070011#include "ext4_jbd2.h"
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -070012#include "ext4_extents.h"
13#include "mballoc.h"
14
15/*
16 * Ext4 Fast Commits
17 * -----------------
18 *
19 * Ext4 fast commits implement fine grained journalling for Ext4.
20 *
21 * Fast commits are organized as a log of tag-length-value (TLV) structs. (See
22 * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by
23 * TLV during the recovery phase. For the scenarios for which we currently
24 * don't have replay code, fast commit falls back to full commits.
25 * Fast commits record delta in one of the following three categories.
26 *
27 * (A) Directory entry updates:
28 *
29 * - EXT4_FC_TAG_UNLINK - records directory entry unlink
30 * - EXT4_FC_TAG_LINK - records directory entry link
31 * - EXT4_FC_TAG_CREAT - records inode and directory entry creation
32 *
33 * (B) File specific data range updates:
34 *
35 * - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode
36 * - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode
37 *
38 * (C) Inode metadata (mtime / ctime etc):
39 *
40 * - EXT4_FC_TAG_INODE - record the inode that should be replayed
41 * during recovery. Note that iblocks field is
42 * not replayed and instead derived during
43 * replay.
44 * Commit Operation
45 * ----------------
46 * With fast commits, we maintain all the directory entry operations in the
47 * order in which they are issued in an in-memory queue. This queue is flushed
48 * to disk during the commit operation. We also maintain a list of inodes
49 * that need to be committed during a fast commit in another in memory queue of
50 * inodes. During the commit operation, we commit in the following order:
51 *
52 * [1] Lock inodes for any further data updates by setting COMMITTING state
53 * [2] Submit data buffers of all the inodes
54 * [3] Wait for [2] to complete
55 * [4] Commit all the directory entry updates in the fast commit space
56 * [5] Commit all the changed inode structures
57 * [6] Write tail tag (this tag ensures the atomicity, please read the following
58 * section for more details).
59 * [7] Wait for [4], [5] and [6] to complete.
60 *
61 * All the inode updates must call ext4_fc_start_update() before starting an
62 * update. If such an ongoing update is present, fast commit waits for it to
63 * complete. The completion of such an update is marked by
64 * ext4_fc_stop_update().
65 *
66 * Fast Commit Ineligibility
67 * -------------------------
68 * Not all operations are supported by fast commits today (e.g extended
69 * attributes). Fast commit ineligiblity is marked by calling one of the
70 * two following functions:
71 *
72 * - ext4_fc_mark_ineligible(): This makes next fast commit operation to fall
73 * back to full commit. This is useful in case of transient errors.
74 *
75 * - ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() - This makes all
76 * the fast commits happening between ext4_fc_start_ineligible() and
77 * ext4_fc_stop_ineligible() and one fast commit after the call to
78 * ext4_fc_stop_ineligible() to fall back to full commits. It is important to
79 * make one more fast commit to fall back to full commit after stop call so
80 * that it guaranteed that the fast commit ineligible operation contained
81 * within ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() is
82 * followed by at least 1 full commit.
83 *
84 * Atomicity of commits
85 * --------------------
Harshad Shirwadkara7407622020-11-05 19:59:03 -080086 * In order to guarantee atomicity during the commit operation, fast commit
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -070087 * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail
88 * tag contains CRC of the contents and TID of the transaction after which
89 * this fast commit should be applied. Recovery code replays fast commit
90 * logs only if there's at least 1 valid tail present. For every fast commit
91 * operation, there is 1 tail. This means, we may end up with multiple tails
92 * in the fast commit space. Here's an example:
93 *
94 * - Create a new file A and remove existing file B
95 * - fsync()
96 * - Append contents to file A
97 * - Truncate file A
98 * - fsync()
99 *
100 * The fast commit space at the end of above operations would look like this:
101 * [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL]
102 * |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->|
103 *
104 * Replay code should thus check for all the valid tails in the FC area.
105 *
106 * TODOs
107 * -----
108 * 1) Make fast commit atomic updates more fine grained. Today, a fast commit
109 * eligible update must be protected within ext4_fc_start_update() and
110 * ext4_fc_stop_update(). These routines are called at much higher
111 * routines. This can be made more fine grained by combining with
112 * ext4_journal_start().
113 *
114 * 2) Same above for ext4_fc_start_ineligible() and ext4_fc_stop_ineligible()
115 *
116 * 3) Handle more ineligible cases.
117 */
118
119#include <trace/events/ext4.h>
120static struct kmem_cache *ext4_fc_dentry_cachep;
121
122static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate)
123{
124 BUFFER_TRACE(bh, "");
125 if (uptodate) {
126 ext4_debug("%s: Block %lld up-to-date",
127 __func__, bh->b_blocknr);
128 set_buffer_uptodate(bh);
129 } else {
130 ext4_debug("%s: Block %lld not up-to-date",
131 __func__, bh->b_blocknr);
132 clear_buffer_uptodate(bh);
133 }
134
135 unlock_buffer(bh);
136}
137
138static inline void ext4_fc_reset_inode(struct inode *inode)
139{
140 struct ext4_inode_info *ei = EXT4_I(inode);
141
142 ei->i_fc_lblk_start = 0;
143 ei->i_fc_lblk_len = 0;
144}
145
146void ext4_fc_init_inode(struct inode *inode)
147{
148 struct ext4_inode_info *ei = EXT4_I(inode);
149
150 ext4_fc_reset_inode(inode);
151 ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING);
152 INIT_LIST_HEAD(&ei->i_fc_list);
153 init_waitqueue_head(&ei->i_fc_wait);
154 atomic_set(&ei->i_fc_updates, 0);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700155}
156
Harshad Shirwadkarf6634e22020-11-05 19:59:02 -0800157/* This function must be called with sbi->s_fc_lock held. */
158static void ext4_fc_wait_committing_inode(struct inode *inode)
Theodore Ts'ofa329e2732020-11-06 23:59:42 -0500159__releases(&EXT4_SB(inode->i_sb)->s_fc_lock)
Harshad Shirwadkarf6634e22020-11-05 19:59:02 -0800160{
161 wait_queue_head_t *wq;
162 struct ext4_inode_info *ei = EXT4_I(inode);
163
164#if (BITS_PER_LONG < 64)
165 DEFINE_WAIT_BIT(wait, &ei->i_state_flags,
166 EXT4_STATE_FC_COMMITTING);
167 wq = bit_waitqueue(&ei->i_state_flags,
168 EXT4_STATE_FC_COMMITTING);
169#else
170 DEFINE_WAIT_BIT(wait, &ei->i_flags,
171 EXT4_STATE_FC_COMMITTING);
172 wq = bit_waitqueue(&ei->i_flags,
173 EXT4_STATE_FC_COMMITTING);
174#endif
175 lockdep_assert_held(&EXT4_SB(inode->i_sb)->s_fc_lock);
176 prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
177 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
178 schedule();
179 finish_wait(wq, &wait.wq_entry);
180}
181
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700182/*
183 * Inform Ext4's fast about start of an inode update
184 *
185 * This function is called by the high level call VFS callbacks before
186 * performing any inode update. This function blocks if there's an ongoing
187 * fast commit on the inode in question.
188 */
189void ext4_fc_start_update(struct inode *inode)
190{
191 struct ext4_inode_info *ei = EXT4_I(inode);
192
Harshad Shirwadkar8016e292020-10-15 13:37:59 -0700193 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
194 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700195 return;
196
197restart:
198 spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
199 if (list_empty(&ei->i_fc_list))
200 goto out;
201
202 if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
Harshad Shirwadkarf6634e22020-11-05 19:59:02 -0800203 ext4_fc_wait_committing_inode(inode);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700204 goto restart;
205 }
206out:
207 atomic_inc(&ei->i_fc_updates);
208 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
209}
210
211/*
212 * Stop inode update and wake up waiting fast commits if any.
213 */
214void ext4_fc_stop_update(struct inode *inode)
215{
216 struct ext4_inode_info *ei = EXT4_I(inode);
217
Harshad Shirwadkar8016e292020-10-15 13:37:59 -0700218 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
219 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700220 return;
221
222 if (atomic_dec_and_test(&ei->i_fc_updates))
223 wake_up_all(&ei->i_fc_wait);
224}
225
226/*
227 * Remove inode from fast commit list. If the inode is being committed
228 * we wait until inode commit is done.
229 */
230void ext4_fc_del(struct inode *inode)
231{
232 struct ext4_inode_info *ei = EXT4_I(inode);
233
Harshad Shirwadkar8016e292020-10-15 13:37:59 -0700234 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
235 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700236 return;
237
238restart:
239 spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
240 if (list_empty(&ei->i_fc_list)) {
241 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
242 return;
243 }
244
245 if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
Harshad Shirwadkarf6634e22020-11-05 19:59:02 -0800246 ext4_fc_wait_committing_inode(inode);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700247 goto restart;
248 }
Harshad Shirwadkarf6634e22020-11-05 19:59:02 -0800249 list_del_init(&ei->i_fc_list);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700250 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
251}
252
253/*
254 * Mark file system as fast commit ineligible. This means that next commit
255 * operation would result in a full jbd2 commit.
256 */
257void ext4_fc_mark_ineligible(struct super_block *sb, int reason)
258{
259 struct ext4_sb_info *sbi = EXT4_SB(sb);
260
Harshad Shirwadkar8016e292020-10-15 13:37:59 -0700261 if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
262 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
263 return;
264
Harshad Shirwadkar9b5f6c92020-11-05 19:59:09 -0800265 ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700266 WARN_ON(reason >= EXT4_FC_REASON_MAX);
267 sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
268}
269
270/*
271 * Start a fast commit ineligible update. Any commits that happen while
272 * such an operation is in progress fall back to full commits.
273 */
274void ext4_fc_start_ineligible(struct super_block *sb, int reason)
275{
276 struct ext4_sb_info *sbi = EXT4_SB(sb);
277
Harshad Shirwadkar8016e292020-10-15 13:37:59 -0700278 if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
279 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
280 return;
281
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700282 WARN_ON(reason >= EXT4_FC_REASON_MAX);
283 sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
284 atomic_inc(&sbi->s_fc_ineligible_updates);
285}
286
287/*
Harshad Shirwadkarababea72020-10-26 21:49:15 -0700288 * Stop a fast commit ineligible update. We set EXT4_MF_FC_INELIGIBLE flag here
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700289 * to ensure that after stopping the ineligible update, at least one full
290 * commit takes place.
291 */
292void ext4_fc_stop_ineligible(struct super_block *sb)
293{
Harshad Shirwadkar8016e292020-10-15 13:37:59 -0700294 if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
295 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
296 return;
297
Harshad Shirwadkar9b5f6c92020-11-05 19:59:09 -0800298 ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700299 atomic_dec(&EXT4_SB(sb)->s_fc_ineligible_updates);
300}
301
302static inline int ext4_fc_is_ineligible(struct super_block *sb)
303{
Harshad Shirwadkar9b5f6c92020-11-05 19:59:09 -0800304 return (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE) ||
305 atomic_read(&EXT4_SB(sb)->s_fc_ineligible_updates));
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700306}
307
308/*
309 * Generic fast commit tracking function. If this is the first time this we are
310 * called after a full commit, we initialize fast commit fields and then call
311 * __fc_track_fn() with update = 0. If we have already been called after a full
312 * commit, we pass update = 1. Based on that, the track function can determine
313 * if it needs to track a field for the first time or if it needs to just
314 * update the previously tracked value.
315 *
316 * If enqueue is set, this function enqueues the inode in fast commit list.
317 */
318static int ext4_fc_track_template(
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800319 handle_t *handle, struct inode *inode,
320 int (*__fc_track_fn)(struct inode *, void *, bool),
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700321 void *args, int enqueue)
322{
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700323 bool update = false;
324 struct ext4_inode_info *ei = EXT4_I(inode);
325 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800326 tid_t tid = 0;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700327 int ret;
328
Harshad Shirwadkar8016e292020-10-15 13:37:59 -0700329 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
330 (sbi->s_mount_state & EXT4_FC_REPLAY))
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700331 return -EOPNOTSUPP;
332
333 if (ext4_fc_is_ineligible(inode->i_sb))
334 return -EINVAL;
335
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800336 tid = handle->h_transaction->t_tid;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700337 mutex_lock(&ei->i_fc_lock);
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800338 if (tid == ei->i_sync_tid) {
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700339 update = true;
340 } else {
341 ext4_fc_reset_inode(inode);
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800342 ei->i_sync_tid = tid;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700343 }
344 ret = __fc_track_fn(inode, args, update);
345 mutex_unlock(&ei->i_fc_lock);
346
347 if (!enqueue)
348 return ret;
349
350 spin_lock(&sbi->s_fc_lock);
351 if (list_empty(&EXT4_I(inode)->i_fc_list))
352 list_add_tail(&EXT4_I(inode)->i_fc_list,
Harshad Shirwadkar9b5f6c92020-11-05 19:59:09 -0800353 (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_COMMITTING)) ?
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700354 &sbi->s_fc_q[FC_Q_STAGING] :
355 &sbi->s_fc_q[FC_Q_MAIN]);
356 spin_unlock(&sbi->s_fc_lock);
357
358 return ret;
359}
360
361struct __track_dentry_update_args {
362 struct dentry *dentry;
363 int op;
364};
365
366/* __track_fn for directory entry updates. Called with ei->i_fc_lock. */
367static int __track_dentry_update(struct inode *inode, void *arg, bool update)
368{
369 struct ext4_fc_dentry_update *node;
370 struct ext4_inode_info *ei = EXT4_I(inode);
371 struct __track_dentry_update_args *dentry_update =
372 (struct __track_dentry_update_args *)arg;
373 struct dentry *dentry = dentry_update->dentry;
374 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
375
376 mutex_unlock(&ei->i_fc_lock);
377 node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);
378 if (!node) {
Harshad Shirwadkarb21ebf12020-11-05 19:58:51 -0800379 ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700380 mutex_lock(&ei->i_fc_lock);
381 return -ENOMEM;
382 }
383
384 node->fcd_op = dentry_update->op;
385 node->fcd_parent = dentry->d_parent->d_inode->i_ino;
386 node->fcd_ino = inode->i_ino;
387 if (dentry->d_name.len > DNAME_INLINE_LEN) {
388 node->fcd_name.name = kmalloc(dentry->d_name.len, GFP_NOFS);
389 if (!node->fcd_name.name) {
390 kmem_cache_free(ext4_fc_dentry_cachep, node);
391 ext4_fc_mark_ineligible(inode->i_sb,
Harshad Shirwadkarb21ebf12020-11-05 19:58:51 -0800392 EXT4_FC_REASON_NOMEM);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700393 mutex_lock(&ei->i_fc_lock);
394 return -ENOMEM;
395 }
396 memcpy((u8 *)node->fcd_name.name, dentry->d_name.name,
397 dentry->d_name.len);
398 } else {
399 memcpy(node->fcd_iname, dentry->d_name.name,
400 dentry->d_name.len);
401 node->fcd_name.name = node->fcd_iname;
402 }
403 node->fcd_name.len = dentry->d_name.len;
404
405 spin_lock(&sbi->s_fc_lock);
Harshad Shirwadkar9b5f6c92020-11-05 19:59:09 -0800406 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_COMMITTING))
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700407 list_add_tail(&node->fcd_list,
408 &sbi->s_fc_dentry_q[FC_Q_STAGING]);
409 else
410 list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]);
411 spin_unlock(&sbi->s_fc_lock);
412 mutex_lock(&ei->i_fc_lock);
413
414 return 0;
415}
416
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800417void __ext4_fc_track_unlink(handle_t *handle,
418 struct inode *inode, struct dentry *dentry)
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700419{
420 struct __track_dentry_update_args args;
421 int ret;
422
423 args.dentry = dentry;
424 args.op = EXT4_FC_TAG_UNLINK;
425
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800426 ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700427 (void *)&args, 0);
428 trace_ext4_fc_track_unlink(inode, dentry, ret);
429}
430
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800431void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry)
432{
433 __ext4_fc_track_unlink(handle, d_inode(dentry), dentry);
434}
435
436void __ext4_fc_track_link(handle_t *handle,
437 struct inode *inode, struct dentry *dentry)
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700438{
439 struct __track_dentry_update_args args;
440 int ret;
441
442 args.dentry = dentry;
443 args.op = EXT4_FC_TAG_LINK;
444
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800445 ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700446 (void *)&args, 0);
447 trace_ext4_fc_track_link(inode, dentry, ret);
448}
449
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800450void ext4_fc_track_link(handle_t *handle, struct dentry *dentry)
451{
452 __ext4_fc_track_link(handle, d_inode(dentry), dentry);
453}
454
Harshad Shirwadkar35ecf662021-03-16 15:19:21 -0700455void __ext4_fc_track_create(handle_t *handle, struct inode *inode,
456 struct dentry *dentry)
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700457{
458 struct __track_dentry_update_args args;
459 int ret;
460
461 args.dentry = dentry;
462 args.op = EXT4_FC_TAG_CREAT;
463
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800464 ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700465 (void *)&args, 0);
466 trace_ext4_fc_track_create(inode, dentry, ret);
467}
468
Harshad Shirwadkar35ecf662021-03-16 15:19:21 -0700469void ext4_fc_track_create(handle_t *handle, struct dentry *dentry)
470{
471 __ext4_fc_track_create(handle, d_inode(dentry), dentry);
472}
473
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700474/* __track_fn for inode tracking */
475static int __track_inode(struct inode *inode, void *arg, bool update)
476{
477 if (update)
478 return -EEXIST;
479
480 EXT4_I(inode)->i_fc_lblk_len = 0;
481
482 return 0;
483}
484
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800485void ext4_fc_track_inode(handle_t *handle, struct inode *inode)
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700486{
487 int ret;
488
489 if (S_ISDIR(inode->i_mode))
490 return;
491
Harshad Shirwadkar556e0312020-11-05 19:59:07 -0800492 if (ext4_should_journal_data(inode)) {
493 ext4_fc_mark_ineligible(inode->i_sb,
494 EXT4_FC_REASON_INODE_JOURNAL_DATA);
495 return;
496 }
497
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800498 ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700499 trace_ext4_fc_track_inode(inode, ret);
500}
501
502struct __track_range_args {
503 ext4_lblk_t start, end;
504};
505
506/* __track_fn for tracking data updates */
507static int __track_range(struct inode *inode, void *arg, bool update)
508{
509 struct ext4_inode_info *ei = EXT4_I(inode);
510 ext4_lblk_t oldstart;
511 struct __track_range_args *__arg =
512 (struct __track_range_args *)arg;
513
514 if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) {
515 ext4_debug("Special inode %ld being modified\n", inode->i_ino);
516 return -ECANCELED;
517 }
518
519 oldstart = ei->i_fc_lblk_start;
520
521 if (update && ei->i_fc_lblk_len > 0) {
522 ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start);
523 ei->i_fc_lblk_len =
524 max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) -
525 ei->i_fc_lblk_start + 1;
526 } else {
527 ei->i_fc_lblk_start = __arg->start;
528 ei->i_fc_lblk_len = __arg->end - __arg->start + 1;
529 }
530
531 return 0;
532}
533
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800534void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start,
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700535 ext4_lblk_t end)
536{
537 struct __track_range_args args;
538 int ret;
539
540 if (S_ISDIR(inode->i_mode))
541 return;
542
543 args.start = start;
544 args.end = end;
545
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -0800546 ret = ext4_fc_track_template(handle, inode, __track_range, &args, 1);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700547
548 trace_ext4_fc_track_range(inode, start, end, ret);
549}
550
551static void ext4_fc_submit_bh(struct super_block *sb)
552{
553 int write_flags = REQ_SYNC;
554 struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;
555
Harshad Shirwadkara7407622020-11-05 19:59:03 -0800556 /* TODO: REQ_FUA | REQ_PREFLUSH is unnecessarily expensive. */
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700557 if (test_opt(sb, BARRIER))
558 write_flags |= REQ_FUA | REQ_PREFLUSH;
559 lock_buffer(bh);
Harshad Shirwadkar764b3fd2020-11-05 19:59:04 -0800560 set_buffer_dirty(bh);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700561 set_buffer_uptodate(bh);
562 bh->b_end_io = ext4_end_buffer_io_sync;
563 submit_bh(REQ_OP_WRITE, write_flags, bh);
564 EXT4_SB(sb)->s_fc_bh = NULL;
565}
566
567/* Ext4 commit path routines */
568
569/* memzero and update CRC */
570static void *ext4_fc_memzero(struct super_block *sb, void *dst, int len,
571 u32 *crc)
572{
573 void *ret;
574
575 ret = memset(dst, 0, len);
576 if (crc)
577 *crc = ext4_chksum(EXT4_SB(sb), *crc, dst, len);
578 return ret;
579}
580
581/*
582 * Allocate len bytes on a fast commit buffer.
583 *
584 * During the commit time this function is used to manage fast commit
585 * block space. We don't split a fast commit log onto different
586 * blocks. So this function makes sure that if there's not enough space
587 * on the current block, the remaining space in the current block is
588 * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case,
589 * new block is from jbd2 and CRC is updated to reflect the padding
590 * we added.
591 */
592static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)
593{
594 struct ext4_fc_tl *tl;
595 struct ext4_sb_info *sbi = EXT4_SB(sb);
596 struct buffer_head *bh;
597 int bsize = sbi->s_journal->j_blocksize;
598 int ret, off = sbi->s_fc_bytes % bsize;
599 int pad_len;
600
601 /*
602 * After allocating len, we should have space at least for a 0 byte
603 * padding.
604 */
605 if (len + sizeof(struct ext4_fc_tl) > bsize)
606 return NULL;
607
608 if (bsize - off - 1 > len + sizeof(struct ext4_fc_tl)) {
609 /*
610 * Only allocate from current buffer if we have enough space for
611 * this request AND we have space to add a zero byte padding.
612 */
613 if (!sbi->s_fc_bh) {
614 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
615 if (ret)
616 return NULL;
617 sbi->s_fc_bh = bh;
618 }
619 sbi->s_fc_bytes += len;
620 return sbi->s_fc_bh->b_data + off;
621 }
622 /* Need to add PAD tag */
623 tl = (struct ext4_fc_tl *)(sbi->s_fc_bh->b_data + off);
624 tl->fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD);
625 pad_len = bsize - off - 1 - sizeof(struct ext4_fc_tl);
626 tl->fc_len = cpu_to_le16(pad_len);
627 if (crc)
628 *crc = ext4_chksum(sbi, *crc, tl, sizeof(*tl));
629 if (pad_len > 0)
630 ext4_fc_memzero(sb, tl + 1, pad_len, crc);
631 ext4_fc_submit_bh(sb);
632
633 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
634 if (ret)
635 return NULL;
636 sbi->s_fc_bh = bh;
637 sbi->s_fc_bytes = (sbi->s_fc_bytes / bsize + 1) * bsize + len;
638 return sbi->s_fc_bh->b_data;
639}
640
641/* memcpy to fc reserved space and update CRC */
642static void *ext4_fc_memcpy(struct super_block *sb, void *dst, const void *src,
643 int len, u32 *crc)
644{
645 if (crc)
646 *crc = ext4_chksum(EXT4_SB(sb), *crc, src, len);
647 return memcpy(dst, src, len);
648}
649
650/*
651 * Complete a fast commit by writing tail tag.
652 *
653 * Writing tail tag marks the end of a fast commit. In order to guarantee
654 * atomicity, after writing tail tag, even if there's space remaining
655 * in the block, next commit shouldn't use it. That's why tail tag
656 * has the length as that of the remaining space on the block.
657 */
658static int ext4_fc_write_tail(struct super_block *sb, u32 crc)
659{
660 struct ext4_sb_info *sbi = EXT4_SB(sb);
661 struct ext4_fc_tl tl;
662 struct ext4_fc_tail tail;
663 int off, bsize = sbi->s_journal->j_blocksize;
664 u8 *dst;
665
666 /*
667 * ext4_fc_reserve_space takes care of allocating an extra block if
668 * there's no enough space on this block for accommodating this tail.
669 */
670 dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(tail), &crc);
671 if (!dst)
672 return -ENOSPC;
673
674 off = sbi->s_fc_bytes % bsize;
675
676 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL);
677 tl.fc_len = cpu_to_le16(bsize - off - 1 + sizeof(struct ext4_fc_tail));
678 sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize);
679
680 ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), &crc);
681 dst += sizeof(tl);
682 tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid);
683 ext4_fc_memcpy(sb, dst, &tail.fc_tid, sizeof(tail.fc_tid), &crc);
684 dst += sizeof(tail.fc_tid);
685 tail.fc_crc = cpu_to_le32(crc);
686 ext4_fc_memcpy(sb, dst, &tail.fc_crc, sizeof(tail.fc_crc), NULL);
687
688 ext4_fc_submit_bh(sb);
689
690 return 0;
691}
692
693/*
694 * Adds tag, length, value and updates CRC. Returns true if tlv was added.
695 * Returns false if there's not enough space.
696 */
697static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val,
698 u32 *crc)
699{
700 struct ext4_fc_tl tl;
701 u8 *dst;
702
703 dst = ext4_fc_reserve_space(sb, sizeof(tl) + len, crc);
704 if (!dst)
705 return false;
706
707 tl.fc_tag = cpu_to_le16(tag);
708 tl.fc_len = cpu_to_le16(len);
709
710 ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc);
711 ext4_fc_memcpy(sb, dst + sizeof(tl), val, len, crc);
712
713 return true;
714}
715
716/* Same as above, but adds dentry tlv. */
717static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u16 tag,
718 int parent_ino, int ino, int dlen,
719 const unsigned char *dname,
720 u32 *crc)
721{
722 struct ext4_fc_dentry_info fcd;
723 struct ext4_fc_tl tl;
724 u8 *dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(fcd) + dlen,
725 crc);
726
727 if (!dst)
728 return false;
729
730 fcd.fc_parent_ino = cpu_to_le32(parent_ino);
731 fcd.fc_ino = cpu_to_le32(ino);
732 tl.fc_tag = cpu_to_le16(tag);
733 tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen);
734 ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc);
735 dst += sizeof(tl);
736 ext4_fc_memcpy(sb, dst, &fcd, sizeof(fcd), crc);
737 dst += sizeof(fcd);
738 ext4_fc_memcpy(sb, dst, dname, dlen, crc);
739 dst += dlen;
740
741 return true;
742}
743
744/*
745 * Writes inode in the fast commit space under TLV with tag @tag.
746 * Returns 0 on success, error on failure.
747 */
748static int ext4_fc_write_inode(struct inode *inode, u32 *crc)
749{
750 struct ext4_inode_info *ei = EXT4_I(inode);
751 int inode_len = EXT4_GOOD_OLD_INODE_SIZE;
752 int ret;
753 struct ext4_iloc iloc;
754 struct ext4_fc_inode fc_inode;
755 struct ext4_fc_tl tl;
756 u8 *dst;
757
758 ret = ext4_get_inode_loc(inode, &iloc);
759 if (ret)
760 return ret;
761
762 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE)
763 inode_len += ei->i_extra_isize;
764
765 fc_inode.fc_ino = cpu_to_le32(inode->i_ino);
766 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE);
767 tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino));
768
Ye Bind575fb52022-09-14 18:08:59 +0800769 ret = -ECANCELED;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700770 dst = ext4_fc_reserve_space(inode->i_sb,
771 sizeof(tl) + inode_len + sizeof(fc_inode.fc_ino), crc);
772 if (!dst)
Ye Bind575fb52022-09-14 18:08:59 +0800773 goto err;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700774
775 if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, sizeof(tl), crc))
Ye Bind575fb52022-09-14 18:08:59 +0800776 goto err;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700777 dst += sizeof(tl);
778 if (!ext4_fc_memcpy(inode->i_sb, dst, &fc_inode, sizeof(fc_inode), crc))
Ye Bind575fb52022-09-14 18:08:59 +0800779 goto err;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700780 dst += sizeof(fc_inode);
781 if (!ext4_fc_memcpy(inode->i_sb, dst, (u8 *)ext4_raw_inode(&iloc),
782 inode_len, crc))
Ye Bind575fb52022-09-14 18:08:59 +0800783 goto err;
784 ret = 0;
785err:
786 brelse(iloc.bh);
787 return ret;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700788}
789
790/*
791 * Writes updated data ranges for the inode in question. Updates CRC.
792 * Returns 0 on success, error otherwise.
793 */
794static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc)
795{
796 ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size;
797 struct ext4_inode_info *ei = EXT4_I(inode);
798 struct ext4_map_blocks map;
799 struct ext4_fc_add_range fc_ext;
800 struct ext4_fc_del_range lrange;
801 struct ext4_extent *ex;
802 int ret;
803
804 mutex_lock(&ei->i_fc_lock);
805 if (ei->i_fc_lblk_len == 0) {
806 mutex_unlock(&ei->i_fc_lock);
807 return 0;
808 }
809 old_blk_size = ei->i_fc_lblk_start;
810 new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1;
811 ei->i_fc_lblk_len = 0;
812 mutex_unlock(&ei->i_fc_lock);
813
814 cur_lblk_off = old_blk_size;
815 jbd_debug(1, "%s: will try writing %d to %d for inode %ld\n",
816 __func__, cur_lblk_off, new_blk_size, inode->i_ino);
817
818 while (cur_lblk_off <= new_blk_size) {
819 map.m_lblk = cur_lblk_off;
820 map.m_len = new_blk_size - cur_lblk_off + 1;
821 ret = ext4_map_blocks(NULL, inode, &map, 0);
822 if (ret < 0)
823 return -ECANCELED;
824
825 if (map.m_len == 0) {
826 cur_lblk_off++;
827 continue;
828 }
829
830 if (ret == 0) {
831 lrange.fc_ino = cpu_to_le32(inode->i_ino);
832 lrange.fc_lblk = cpu_to_le32(map.m_lblk);
833 lrange.fc_len = cpu_to_le32(map.m_len);
834 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE,
835 sizeof(lrange), (u8 *)&lrange, crc))
836 return -ENOSPC;
837 } else {
Hou Taodc094212021-08-20 12:45:05 +0800838 unsigned int max = (map.m_flags & EXT4_MAP_UNWRITTEN) ?
839 EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN;
840
841 /* Limit the number of blocks in one extent */
842 map.m_len = min(max, map.m_len);
843
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700844 fc_ext.fc_ino = cpu_to_le32(inode->i_ino);
845 ex = (struct ext4_extent *)&fc_ext.fc_ex;
846 ex->ee_block = cpu_to_le32(map.m_lblk);
847 ex->ee_len = cpu_to_le16(map.m_len);
848 ext4_ext_store_pblock(ex, map.m_pblk);
849 if (map.m_flags & EXT4_MAP_UNWRITTEN)
850 ext4_ext_mark_unwritten(ex);
851 else
852 ext4_ext_mark_initialized(ex);
853 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE,
854 sizeof(fc_ext), (u8 *)&fc_ext, crc))
855 return -ENOSPC;
856 }
857
858 cur_lblk_off += map.m_len;
859 }
860
861 return 0;
862}
863
864
865/* Submit data for all the fast commit inodes */
866static int ext4_fc_submit_inode_data_all(journal_t *journal)
867{
868 struct super_block *sb = (struct super_block *)(journal->j_private);
869 struct ext4_sb_info *sbi = EXT4_SB(sb);
870 struct ext4_inode_info *ei;
871 struct list_head *pos;
872 int ret = 0;
873
874 spin_lock(&sbi->s_fc_lock);
Harshad Shirwadkar9b5f6c92020-11-05 19:59:09 -0800875 ext4_set_mount_flag(sb, EXT4_MF_FC_COMMITTING);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700876 list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) {
877 ei = list_entry(pos, struct ext4_inode_info, i_fc_list);
878 ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING);
879 while (atomic_read(&ei->i_fc_updates)) {
880 DEFINE_WAIT(wait);
881
882 prepare_to_wait(&ei->i_fc_wait, &wait,
883 TASK_UNINTERRUPTIBLE);
884 if (atomic_read(&ei->i_fc_updates)) {
885 spin_unlock(&sbi->s_fc_lock);
886 schedule();
887 spin_lock(&sbi->s_fc_lock);
888 }
889 finish_wait(&ei->i_fc_wait, &wait);
890 }
891 spin_unlock(&sbi->s_fc_lock);
892 ret = jbd2_submit_inode_data(ei->jinode);
893 if (ret)
894 return ret;
895 spin_lock(&sbi->s_fc_lock);
896 }
897 spin_unlock(&sbi->s_fc_lock);
898
899 return ret;
900}
901
902/* Wait for completion of data for all the fast commit inodes */
903static int ext4_fc_wait_inode_data_all(journal_t *journal)
904{
905 struct super_block *sb = (struct super_block *)(journal->j_private);
906 struct ext4_sb_info *sbi = EXT4_SB(sb);
907 struct ext4_inode_info *pos, *n;
908 int ret = 0;
909
910 spin_lock(&sbi->s_fc_lock);
911 list_for_each_entry_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
912 if (!ext4_test_inode_state(&pos->vfs_inode,
913 EXT4_STATE_FC_COMMITTING))
914 continue;
915 spin_unlock(&sbi->s_fc_lock);
916
917 ret = jbd2_wait_inode_data(journal, pos->jinode);
918 if (ret)
919 return ret;
920 spin_lock(&sbi->s_fc_lock);
921 }
922 spin_unlock(&sbi->s_fc_lock);
923
924 return 0;
925}
926
927/* Commit all the directory entry updates */
928static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc)
Theodore Ts'ofa329e2732020-11-06 23:59:42 -0500929__acquires(&sbi->s_fc_lock)
930__releases(&sbi->s_fc_lock)
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700931{
932 struct super_block *sb = (struct super_block *)(journal->j_private);
933 struct ext4_sb_info *sbi = EXT4_SB(sb);
934 struct ext4_fc_dentry_update *fc_dentry;
935 struct inode *inode;
936 struct list_head *pos, *n, *fcd_pos, *fcd_n;
937 struct ext4_inode_info *ei;
938 int ret;
939
940 if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN]))
941 return 0;
942 list_for_each_safe(fcd_pos, fcd_n, &sbi->s_fc_dentry_q[FC_Q_MAIN]) {
943 fc_dentry = list_entry(fcd_pos, struct ext4_fc_dentry_update,
944 fcd_list);
945 if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) {
946 spin_unlock(&sbi->s_fc_lock);
947 if (!ext4_fc_add_dentry_tlv(
948 sb, fc_dentry->fcd_op,
949 fc_dentry->fcd_parent, fc_dentry->fcd_ino,
950 fc_dentry->fcd_name.len,
951 fc_dentry->fcd_name.name, crc)) {
952 ret = -ENOSPC;
953 goto lock_and_exit;
954 }
955 spin_lock(&sbi->s_fc_lock);
956 continue;
957 }
958
959 inode = NULL;
960 list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) {
961 ei = list_entry(pos, struct ext4_inode_info, i_fc_list);
962 if (ei->vfs_inode.i_ino == fc_dentry->fcd_ino) {
963 inode = &ei->vfs_inode;
964 break;
965 }
966 }
967 /*
968 * If we don't find inode in our list, then it was deleted,
969 * in which case, we don't need to record it's create tag.
970 */
971 if (!inode)
972 continue;
973 spin_unlock(&sbi->s_fc_lock);
974
975 /*
976 * We first write the inode and then the create dirent. This
977 * allows the recovery code to create an unnamed inode first
978 * and then link it to a directory entry. This allows us
979 * to use namei.c routines almost as is and simplifies
980 * the recovery code.
981 */
982 ret = ext4_fc_write_inode(inode, crc);
983 if (ret)
984 goto lock_and_exit;
985
986 ret = ext4_fc_write_inode_data(inode, crc);
987 if (ret)
988 goto lock_and_exit;
989
990 if (!ext4_fc_add_dentry_tlv(
991 sb, fc_dentry->fcd_op,
992 fc_dentry->fcd_parent, fc_dentry->fcd_ino,
993 fc_dentry->fcd_name.len,
994 fc_dentry->fcd_name.name, crc)) {
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700995 ret = -ENOSPC;
996 goto lock_and_exit;
997 }
998
999 spin_lock(&sbi->s_fc_lock);
1000 }
1001 return 0;
1002lock_and_exit:
1003 spin_lock(&sbi->s_fc_lock);
1004 return ret;
1005}
1006
1007static int ext4_fc_perform_commit(journal_t *journal)
1008{
1009 struct super_block *sb = (struct super_block *)(journal->j_private);
1010 struct ext4_sb_info *sbi = EXT4_SB(sb);
1011 struct ext4_inode_info *iter;
1012 struct ext4_fc_head head;
1013 struct list_head *pos;
1014 struct inode *inode;
1015 struct blk_plug plug;
1016 int ret = 0;
1017 u32 crc = 0;
1018
1019 ret = ext4_fc_submit_inode_data_all(journal);
1020 if (ret)
1021 return ret;
1022
1023 ret = ext4_fc_wait_inode_data_all(journal);
1024 if (ret)
1025 return ret;
1026
Harshad Shirwadkarda0c5d22020-11-05 19:59:08 -08001027 /*
1028 * If file system device is different from journal device, issue a cache
1029 * flush before we start writing fast commit blocks.
1030 */
1031 if (journal->j_fs_dev != journal->j_dev)
1032 blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS);
1033
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07001034 blk_start_plug(&plug);
1035 if (sbi->s_fc_bytes == 0) {
1036 /*
1037 * Add a head tag only if this is the first fast commit
1038 * in this TID.
1039 */
1040 head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES);
1041 head.fc_tid = cpu_to_le32(
1042 sbi->s_journal->j_running_transaction->t_tid);
1043 if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head),
Xu Yihang72447c92021-04-08 15:00:33 +08001044 (u8 *)&head, &crc)) {
1045 ret = -ENOSPC;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07001046 goto out;
Xu Yihang72447c92021-04-08 15:00:33 +08001047 }
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07001048 }
1049
1050 spin_lock(&sbi->s_fc_lock);
1051 ret = ext4_fc_commit_dentry_updates(journal, &crc);
1052 if (ret) {
1053 spin_unlock(&sbi->s_fc_lock);
1054 goto out;
1055 }
1056
1057 list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) {
1058 iter = list_entry(pos, struct ext4_inode_info, i_fc_list);
1059 inode = &iter->vfs_inode;
1060 if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING))
1061 continue;
1062
1063 spin_unlock(&sbi->s_fc_lock);
1064 ret = ext4_fc_write_inode_data(inode, &crc);
1065 if (ret)
1066 goto out;
1067 ret = ext4_fc_write_inode(inode, &crc);
1068 if (ret)
1069 goto out;
1070 spin_lock(&sbi->s_fc_lock);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07001071 }
1072 spin_unlock(&sbi->s_fc_lock);
1073
1074 ret = ext4_fc_write_tail(sb, crc);
1075
1076out:
1077 blk_finish_plug(&plug);
1078 return ret;
1079}
1080
1081/*
1082 * The main commit entry point. Performs a fast commit for transaction
1083 * commit_tid if needed. If it's not possible to perform a fast commit
1084 * due to various reasons, we fall back to full commit. Returns 0
1085 * on success, error otherwise.
1086 */
1087int ext4_fc_commit(journal_t *journal, tid_t commit_tid)
1088{
1089 struct super_block *sb = (struct super_block *)(journal->j_private);
1090 struct ext4_sb_info *sbi = EXT4_SB(sb);
1091 int nblks = 0, ret, bsize = journal->j_blocksize;
1092 int subtid = atomic_read(&sbi->s_fc_subtid);
1093 int reason = EXT4_FC_REASON_OK, fc_bufs_before = 0;
1094 ktime_t start_time, commit_time;
1095
1096 trace_ext4_fc_commit_start(sb);
1097
1098 start_time = ktime_get();
1099
1100 if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
1101 (ext4_fc_is_ineligible(sb))) {
1102 reason = EXT4_FC_REASON_INELIGIBLE;
1103 goto out;
1104 }
1105
1106restart_fc:
1107 ret = jbd2_fc_begin_commit(journal, commit_tid);
1108 if (ret == -EALREADY) {
1109 /* There was an ongoing commit, check if we need to restart */
1110 if (atomic_read(&sbi->s_fc_subtid) <= subtid &&
1111 commit_tid > journal->j_commit_sequence)
1112 goto restart_fc;
1113 reason = EXT4_FC_REASON_ALREADY_COMMITTED;
1114 goto out;
1115 } else if (ret) {
1116 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1117 reason = EXT4_FC_REASON_FC_START_FAILED;
1118 goto out;
1119 }
1120
1121 fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize;
1122 ret = ext4_fc_perform_commit(journal);
1123 if (ret < 0) {
1124 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1125 reason = EXT4_FC_REASON_FC_FAILED;
1126 goto out;
1127 }
1128 nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before;
1129 ret = jbd2_fc_wait_bufs(journal, nblks);
1130 if (ret < 0) {
1131 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1132 reason = EXT4_FC_REASON_FC_FAILED;
1133 goto out;
1134 }
1135 atomic_inc(&sbi->s_fc_subtid);
1136 jbd2_fc_end_commit(journal);
1137out:
1138 /* Has any ineligible update happened since we started? */
1139 if (reason == EXT4_FC_REASON_OK && ext4_fc_is_ineligible(sb)) {
1140 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1141 reason = EXT4_FC_REASON_INELIGIBLE;
1142 }
1143
1144 spin_lock(&sbi->s_fc_lock);
1145 if (reason != EXT4_FC_REASON_OK &&
1146 reason != EXT4_FC_REASON_ALREADY_COMMITTED) {
1147 sbi->s_fc_stats.fc_ineligible_commits++;
1148 } else {
1149 sbi->s_fc_stats.fc_num_commits++;
1150 sbi->s_fc_stats.fc_numblks += nblks;
1151 }
1152 spin_unlock(&sbi->s_fc_lock);
1153 nblks = (reason == EXT4_FC_REASON_OK) ? nblks : 0;
1154 trace_ext4_fc_commit_stop(sb, nblks, reason);
1155 commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time));
1156 /*
1157 * weight the commit time higher than the average time so we don't
1158 * react too strongly to vast changes in the commit time
1159 */
1160 if (likely(sbi->s_fc_avg_commit_time))
1161 sbi->s_fc_avg_commit_time = (commit_time +
1162 sbi->s_fc_avg_commit_time * 3) / 4;
1163 else
1164 sbi->s_fc_avg_commit_time = commit_time;
1165 jbd_debug(1,
1166 "Fast commit ended with blks = %d, reason = %d, subtid - %d",
1167 nblks, reason, subtid);
1168 if (reason == EXT4_FC_REASON_FC_FAILED)
Harshad Shirwadkar0bce5772020-11-05 19:58:58 -08001169 return jbd2_fc_end_commit_fallback(journal);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07001170 if (reason == EXT4_FC_REASON_FC_START_FAILED ||
1171 reason == EXT4_FC_REASON_INELIGIBLE)
1172 return jbd2_complete_transaction(journal, commit_tid);
1173 return 0;
1174}
1175
Harshad Shirwadkarff780b92020-10-15 13:37:56 -07001176/*
1177 * Fast commit cleanup routine. This is called after every fast commit and
1178 * full commit. full is true if we are called after a full commit.
1179 */
1180static void ext4_fc_cleanup(journal_t *journal, int full)
1181{
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07001182 struct super_block *sb = journal->j_private;
1183 struct ext4_sb_info *sbi = EXT4_SB(sb);
1184 struct ext4_inode_info *iter;
1185 struct ext4_fc_dentry_update *fc_dentry;
1186 struct list_head *pos, *n;
1187
1188 if (full && sbi->s_fc_bh)
1189 sbi->s_fc_bh = NULL;
1190
1191 jbd2_fc_release_bufs(journal);
1192
1193 spin_lock(&sbi->s_fc_lock);
1194 list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) {
1195 iter = list_entry(pos, struct ext4_inode_info, i_fc_list);
1196 list_del_init(&iter->i_fc_list);
1197 ext4_clear_inode_state(&iter->vfs_inode,
1198 EXT4_STATE_FC_COMMITTING);
1199 ext4_fc_reset_inode(&iter->vfs_inode);
1200 /* Make sure EXT4_STATE_FC_COMMITTING bit is clear */
1201 smp_mb();
1202#if (BITS_PER_LONG < 64)
1203 wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING);
1204#else
1205 wake_up_bit(&iter->i_flags, EXT4_STATE_FC_COMMITTING);
1206#endif
1207 }
1208
1209 while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) {
1210 fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN],
1211 struct ext4_fc_dentry_update,
1212 fcd_list);
1213 list_del_init(&fc_dentry->fcd_list);
1214 spin_unlock(&sbi->s_fc_lock);
1215
1216 if (fc_dentry->fcd_name.name &&
1217 fc_dentry->fcd_name.len > DNAME_INLINE_LEN)
1218 kfree(fc_dentry->fcd_name.name);
1219 kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);
1220 spin_lock(&sbi->s_fc_lock);
1221 }
1222
1223 list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING],
1224 &sbi->s_fc_dentry_q[FC_Q_MAIN]);
1225 list_splice_init(&sbi->s_fc_q[FC_Q_STAGING],
Daejun Park15a062c2020-12-30 18:48:51 +09001226 &sbi->s_fc_q[FC_Q_MAIN]);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07001227
Harshad Shirwadkar9b5f6c92020-11-05 19:59:09 -08001228 ext4_clear_mount_flag(sb, EXT4_MF_FC_COMMITTING);
1229 ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07001230
1231 if (full)
1232 sbi->s_fc_bytes = 0;
1233 spin_unlock(&sbi->s_fc_lock);
1234 trace_ext4_fc_stats(sb);
Harshad Shirwadkarff780b92020-10-15 13:37:56 -07001235}
Harshad Shirwadkar6866d7b2020-10-15 13:37:55 -07001236
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001237/* Ext4 Replay Path Routines */
1238
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001239/* Helper struct for dentry replay routines */
1240struct dentry_info_args {
1241 int parent_ino, dname_len, ino, inode_len;
1242 char *dname;
1243};
1244
1245static inline void tl_to_darg(struct dentry_info_args *darg,
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001246 struct ext4_fc_tl *tl, u8 *val)
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001247{
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001248 struct ext4_fc_dentry_info fcd;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001249
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001250 memcpy(&fcd, val, sizeof(fcd));
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001251
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001252 darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino);
1253 darg->ino = le32_to_cpu(fcd.fc_ino);
1254 darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname);
1255 darg->dname_len = le16_to_cpu(tl->fc_len) -
1256 sizeof(struct ext4_fc_dentry_info);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001257}
1258
1259/* Unlink replay function */
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001260static int ext4_fc_replay_unlink(struct super_block *sb, struct ext4_fc_tl *tl,
1261 u8 *val)
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001262{
1263 struct inode *inode, *old_parent;
1264 struct qstr entry;
1265 struct dentry_info_args darg;
1266 int ret = 0;
1267
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001268 tl_to_darg(&darg, tl, val);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001269
1270 trace_ext4_fc_replay(sb, EXT4_FC_TAG_UNLINK, darg.ino,
1271 darg.parent_ino, darg.dname_len);
1272
1273 entry.name = darg.dname;
1274 entry.len = darg.dname_len;
1275 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1276
Yi Li6c557cb12020-12-30 11:38:27 +08001277 if (IS_ERR(inode)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001278 jbd_debug(1, "Inode %d not found", darg.ino);
1279 return 0;
1280 }
1281
1282 old_parent = ext4_iget(sb, darg.parent_ino,
1283 EXT4_IGET_NORMAL);
Yi Li6c557cb12020-12-30 11:38:27 +08001284 if (IS_ERR(old_parent)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001285 jbd_debug(1, "Dir with inode %d not found", darg.parent_ino);
1286 iput(inode);
1287 return 0;
1288 }
1289
Harshad Shirwadkara80f7fc2020-11-05 19:58:53 -08001290 ret = __ext4_unlink(NULL, old_parent, &entry, inode);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001291 /* -ENOENT ok coz it might not exist anymore. */
1292 if (ret == -ENOENT)
1293 ret = 0;
1294 iput(old_parent);
1295 iput(inode);
1296 return ret;
1297}
1298
1299static int ext4_fc_replay_link_internal(struct super_block *sb,
1300 struct dentry_info_args *darg,
1301 struct inode *inode)
1302{
1303 struct inode *dir = NULL;
1304 struct dentry *dentry_dir = NULL, *dentry_inode = NULL;
1305 struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len);
1306 int ret = 0;
1307
1308 dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL);
1309 if (IS_ERR(dir)) {
1310 jbd_debug(1, "Dir with inode %d not found.", darg->parent_ino);
1311 dir = NULL;
1312 goto out;
1313 }
1314
1315 dentry_dir = d_obtain_alias(dir);
1316 if (IS_ERR(dentry_dir)) {
1317 jbd_debug(1, "Failed to obtain dentry");
1318 dentry_dir = NULL;
1319 goto out;
1320 }
1321
1322 dentry_inode = d_alloc(dentry_dir, &qstr_dname);
1323 if (!dentry_inode) {
1324 jbd_debug(1, "Inode dentry not created.");
1325 ret = -ENOMEM;
1326 goto out;
1327 }
1328
1329 ret = __ext4_link(dir, inode, dentry_inode);
1330 /*
1331 * It's possible that link already existed since data blocks
1332 * for the dir in question got persisted before we crashed OR
1333 * we replayed this tag and crashed before the entire replay
1334 * could complete.
1335 */
1336 if (ret && ret != -EEXIST) {
1337 jbd_debug(1, "Failed to link\n");
1338 goto out;
1339 }
1340
1341 ret = 0;
1342out:
1343 if (dentry_dir) {
1344 d_drop(dentry_dir);
1345 dput(dentry_dir);
1346 } else if (dir) {
1347 iput(dir);
1348 }
1349 if (dentry_inode) {
1350 d_drop(dentry_inode);
1351 dput(dentry_inode);
1352 }
1353
1354 return ret;
1355}
1356
1357/* Link replay function */
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001358static int ext4_fc_replay_link(struct super_block *sb, struct ext4_fc_tl *tl,
1359 u8 *val)
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001360{
1361 struct inode *inode;
1362 struct dentry_info_args darg;
1363 int ret = 0;
1364
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001365 tl_to_darg(&darg, tl, val);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001366 trace_ext4_fc_replay(sb, EXT4_FC_TAG_LINK, darg.ino,
1367 darg.parent_ino, darg.dname_len);
1368
1369 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
Yi Li6c557cb12020-12-30 11:38:27 +08001370 if (IS_ERR(inode)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001371 jbd_debug(1, "Inode not found.");
1372 return 0;
1373 }
1374
1375 ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1376 iput(inode);
1377 return ret;
1378}
1379
1380/*
1381 * Record all the modified inodes during replay. We use this later to setup
1382 * block bitmaps correctly.
1383 */
1384static int ext4_fc_record_modified_inode(struct super_block *sb, int ino)
1385{
1386 struct ext4_fc_replay_state *state;
1387 int i;
1388
1389 state = &EXT4_SB(sb)->s_fc_replay_state;
1390 for (i = 0; i < state->fc_modified_inodes_used; i++)
1391 if (state->fc_modified_inodes[i] == ino)
1392 return 0;
1393 if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) {
Ye Binc9ce7762022-09-21 14:40:38 +08001394 int *fc_modified_inodes;
1395
1396 fc_modified_inodes = krealloc(state->fc_modified_inodes,
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301397 sizeof(int) * (state->fc_modified_inodes_size +
1398 EXT4_FC_REPLAY_REALLOC_INCREMENT),
1399 GFP_KERNEL);
Ye Binc9ce7762022-09-21 14:40:38 +08001400 if (!fc_modified_inodes)
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001401 return -ENOMEM;
Ye Binc9ce7762022-09-21 14:40:38 +08001402 state->fc_modified_inodes = fc_modified_inodes;
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301403 state->fc_modified_inodes_size +=
1404 EXT4_FC_REPLAY_REALLOC_INCREMENT;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001405 }
1406 state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino;
1407 return 0;
1408}
1409
1410/*
1411 * Inode replay function
1412 */
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001413static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl,
1414 u8 *val)
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001415{
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001416 struct ext4_fc_inode fc_inode;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001417 struct ext4_inode *raw_inode;
1418 struct ext4_inode *raw_fc_inode;
1419 struct inode *inode = NULL;
1420 struct ext4_iloc iloc;
1421 int inode_len, ino, ret, tag = le16_to_cpu(tl->fc_tag);
1422 struct ext4_extent_header *eh;
1423
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001424 memcpy(&fc_inode, val, sizeof(fc_inode));
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001425
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001426 ino = le32_to_cpu(fc_inode.fc_ino);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001427 trace_ext4_fc_replay(sb, tag, ino, 0, 0);
1428
1429 inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
Yi Li6c557cb12020-12-30 11:38:27 +08001430 if (!IS_ERR(inode)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001431 ext4_ext_clear_bb(inode);
1432 iput(inode);
1433 }
Yi Li6c557cb12020-12-30 11:38:27 +08001434 inode = NULL;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001435
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301436 ret = ext4_fc_record_modified_inode(sb, ino);
1437 if (ret)
1438 goto out;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001439
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001440 raw_fc_inode = (struct ext4_inode *)
1441 (val + offsetof(struct ext4_fc_inode, fc_raw_inode));
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001442 ret = ext4_get_fc_inode_loc(sb, ino, &iloc);
1443 if (ret)
1444 goto out;
1445
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001446 inode_len = le16_to_cpu(tl->fc_len) - sizeof(struct ext4_fc_inode);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001447 raw_inode = ext4_raw_inode(&iloc);
1448
1449 memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block));
1450 memcpy(&raw_inode->i_generation, &raw_fc_inode->i_generation,
1451 inode_len - offsetof(struct ext4_inode, i_generation));
1452 if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) {
1453 eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]);
1454 if (eh->eh_magic != EXT4_EXT_MAGIC) {
1455 memset(eh, 0, sizeof(*eh));
1456 eh->eh_magic = EXT4_EXT_MAGIC;
1457 eh->eh_max = cpu_to_le16(
1458 (sizeof(raw_inode->i_block) -
1459 sizeof(struct ext4_extent_header))
1460 / sizeof(struct ext4_extent));
1461 }
1462 } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) {
1463 memcpy(raw_inode->i_block, raw_fc_inode->i_block,
1464 sizeof(raw_inode->i_block));
1465 }
1466
1467 /* Immediately update the inode on disk. */
1468 ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1469 if (ret)
1470 goto out;
1471 ret = sync_dirty_buffer(iloc.bh);
1472 if (ret)
1473 goto out;
1474 ret = ext4_mark_inode_used(sb, ino);
1475 if (ret)
1476 goto out;
1477
1478 /* Given that we just wrote the inode on disk, this SHOULD succeed. */
1479 inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
Yi Li6c557cb12020-12-30 11:38:27 +08001480 if (IS_ERR(inode)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001481 jbd_debug(1, "Inode not found.");
1482 return -EFSCORRUPTED;
1483 }
1484
1485 /*
1486 * Our allocator could have made different decisions than before
1487 * crashing. This should be fixed but until then, we calculate
1488 * the number of blocks the inode.
1489 */
1490 ext4_ext_replay_set_iblocks(inode);
1491
1492 inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation);
1493 ext4_reset_inode_seed(inode);
1494
1495 ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode));
1496 ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1497 sync_dirty_buffer(iloc.bh);
1498 brelse(iloc.bh);
1499out:
1500 iput(inode);
1501 if (!ret)
1502 blkdev_issue_flush(sb->s_bdev, GFP_KERNEL);
1503
1504 return 0;
1505}
1506
1507/*
1508 * Dentry create replay function.
1509 *
1510 * EXT4_FC_TAG_CREAT is preceded by EXT4_FC_TAG_INODE_FULL. Which means, the
1511 * inode for which we are trying to create a dentry here, should already have
1512 * been replayed before we start here.
1513 */
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001514static int ext4_fc_replay_create(struct super_block *sb, struct ext4_fc_tl *tl,
1515 u8 *val)
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001516{
1517 int ret = 0;
1518 struct inode *inode = NULL;
1519 struct inode *dir = NULL;
1520 struct dentry_info_args darg;
1521
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001522 tl_to_darg(&darg, tl, val);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001523
1524 trace_ext4_fc_replay(sb, EXT4_FC_TAG_CREAT, darg.ino,
1525 darg.parent_ino, darg.dname_len);
1526
1527 /* This takes care of update group descriptor and other metadata */
1528 ret = ext4_mark_inode_used(sb, darg.ino);
1529 if (ret)
1530 goto out;
1531
1532 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
Yi Li6c557cb12020-12-30 11:38:27 +08001533 if (IS_ERR(inode)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001534 jbd_debug(1, "inode %d not found.", darg.ino);
1535 inode = NULL;
1536 ret = -EINVAL;
1537 goto out;
1538 }
1539
1540 if (S_ISDIR(inode->i_mode)) {
1541 /*
1542 * If we are creating a directory, we need to make sure that the
1543 * dot and dot dot dirents are setup properly.
1544 */
1545 dir = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL);
Yi Li6c557cb12020-12-30 11:38:27 +08001546 if (IS_ERR(dir)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001547 jbd_debug(1, "Dir %d not found.", darg.ino);
1548 goto out;
1549 }
1550 ret = ext4_init_new_dir(NULL, dir, inode);
1551 iput(dir);
1552 if (ret) {
1553 ret = 0;
1554 goto out;
1555 }
1556 }
1557 ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1558 if (ret)
1559 goto out;
1560 set_nlink(inode, 1);
1561 ext4_mark_inode_dirty(NULL, inode);
1562out:
1563 if (inode)
1564 iput(inode);
1565 return ret;
1566}
1567
1568/*
Xin Yin8d71fc22022-01-10 11:51:40 +08001569 * Record physical disk regions which are in use as per fast commit area,
1570 * and used by inodes during replay phase. Our simple replay phase
1571 * allocator excludes these regions from allocation.
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001572 */
Xin Yin8d71fc22022-01-10 11:51:40 +08001573int ext4_fc_record_regions(struct super_block *sb, int ino,
1574 ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay)
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001575{
1576 struct ext4_fc_replay_state *state;
1577 struct ext4_fc_alloc_region *region;
1578
1579 state = &EXT4_SB(sb)->s_fc_replay_state;
Xin Yin8d71fc22022-01-10 11:51:40 +08001580 /*
1581 * during replay phase, the fc_regions_valid may not same as
1582 * fc_regions_used, update it when do new additions.
1583 */
1584 if (replay && state->fc_regions_used != state->fc_regions_valid)
1585 state->fc_regions_used = state->fc_regions_valid;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001586 if (state->fc_regions_used == state->fc_regions_size) {
Ye Bin2cfb7692022-09-21 14:40:39 +08001587 struct ext4_fc_alloc_region *fc_regions;
1588
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001589 state->fc_regions_size +=
1590 EXT4_FC_REPLAY_REALLOC_INCREMENT;
Ye Bin2cfb7692022-09-21 14:40:39 +08001591 fc_regions = krealloc(state->fc_regions,
1592 state->fc_regions_size *
1593 sizeof(struct ext4_fc_alloc_region),
1594 GFP_KERNEL);
1595 if (!fc_regions)
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001596 return -ENOMEM;
Ye Bin2cfb7692022-09-21 14:40:39 +08001597 state->fc_regions = fc_regions;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001598 }
1599 region = &state->fc_regions[state->fc_regions_used++];
1600 region->ino = ino;
1601 region->lblk = lblk;
1602 region->pblk = pblk;
1603 region->len = len;
1604
Xin Yin8d71fc22022-01-10 11:51:40 +08001605 if (replay)
1606 state->fc_regions_valid++;
1607
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001608 return 0;
1609}
1610
1611/* Replay add range tag */
1612static int ext4_fc_replay_add_range(struct super_block *sb,
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001613 struct ext4_fc_tl *tl, u8 *val)
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001614{
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001615 struct ext4_fc_add_range fc_add_ex;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001616 struct ext4_extent newex, *ex;
1617 struct inode *inode;
1618 ext4_lblk_t start, cur;
1619 int remaining, len;
1620 ext4_fsblk_t start_pblk;
1621 struct ext4_map_blocks map;
1622 struct ext4_ext_path *path = NULL;
1623 int ret;
1624
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001625 memcpy(&fc_add_ex, val, sizeof(fc_add_ex));
1626 ex = (struct ext4_extent *)&fc_add_ex.fc_ex;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001627
1628 trace_ext4_fc_replay(sb, EXT4_FC_TAG_ADD_RANGE,
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001629 le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block),
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001630 ext4_ext_get_actual_len(ex));
1631
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001632 inode = ext4_iget(sb, le32_to_cpu(fc_add_ex.fc_ino), EXT4_IGET_NORMAL);
Yi Li6c557cb12020-12-30 11:38:27 +08001633 if (IS_ERR(inode)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001634 jbd_debug(1, "Inode not found.");
1635 return 0;
1636 }
1637
1638 ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301639 if (ret)
1640 goto out;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001641
1642 start = le32_to_cpu(ex->ee_block);
1643 start_pblk = ext4_ext_pblock(ex);
1644 len = ext4_ext_get_actual_len(ex);
1645
1646 cur = start;
1647 remaining = len;
1648 jbd_debug(1, "ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %ld\n",
1649 start, start_pblk, len, ext4_ext_is_unwritten(ex),
1650 inode->i_ino);
1651
1652 while (remaining > 0) {
1653 map.m_lblk = cur;
1654 map.m_len = remaining;
1655 map.m_pblk = 0;
1656 ret = ext4_map_blocks(NULL, inode, &map, 0);
1657
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301658 if (ret < 0)
1659 goto out;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001660
1661 if (ret == 0) {
1662 /* Range is not mapped */
1663 path = ext4_find_extent(inode, cur, NULL, 0);
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301664 if (IS_ERR(path))
1665 goto out;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001666 memset(&newex, 0, sizeof(newex));
1667 newex.ee_block = cpu_to_le32(cur);
1668 ext4_ext_store_pblock(
1669 &newex, start_pblk + cur - start);
1670 newex.ee_len = cpu_to_le16(map.m_len);
1671 if (ext4_ext_is_unwritten(ex))
1672 ext4_ext_mark_unwritten(&newex);
1673 down_write(&EXT4_I(inode)->i_data_sem);
1674 ret = ext4_ext_insert_extent(
1675 NULL, inode, &path, &newex, 0);
1676 up_write((&EXT4_I(inode)->i_data_sem));
1677 ext4_ext_drop_refs(path);
1678 kfree(path);
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301679 if (ret)
1680 goto out;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001681 goto next;
1682 }
1683
1684 if (start_pblk + cur - start != map.m_pblk) {
1685 /*
1686 * Logical to physical mapping changed. This can happen
1687 * if this range was removed and then reallocated to
1688 * map to new physical blocks during a fast commit.
1689 */
1690 ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1691 ext4_ext_is_unwritten(ex),
1692 start_pblk + cur - start);
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301693 if (ret)
1694 goto out;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001695 /*
1696 * Mark the old blocks as free since they aren't used
1697 * anymore. We maintain an array of all the modified
1698 * inodes. In case these blocks are still used at either
1699 * a different logical range in the same inode or in
1700 * some different inode, we will mark them as allocated
1701 * at the end of the FC replay using our array of
1702 * modified inodes.
1703 */
1704 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0);
1705 goto next;
1706 }
1707
1708 /* Range is mapped and needs a state change */
Arnd Bergmann054add22021-04-09 22:12:05 +02001709 jbd_debug(1, "Converting from %ld to %d %lld",
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001710 map.m_flags & EXT4_MAP_UNWRITTEN,
1711 ext4_ext_is_unwritten(ex), map.m_pblk);
1712 ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1713 ext4_ext_is_unwritten(ex), map.m_pblk);
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301714 if (ret)
1715 goto out;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001716 /*
1717 * We may have split the extent tree while toggling the state.
1718 * Try to shrink the extent tree now.
1719 */
1720 ext4_ext_replay_shrink_inode(inode, start + len);
1721next:
1722 cur += map.m_len;
1723 remaining -= map.m_len;
1724 }
1725 ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >>
1726 sb->s_blocksize_bits);
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301727out:
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001728 iput(inode);
1729 return 0;
1730}
1731
1732/* Replay DEL_RANGE tag */
1733static int
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001734ext4_fc_replay_del_range(struct super_block *sb, struct ext4_fc_tl *tl,
1735 u8 *val)
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001736{
1737 struct inode *inode;
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001738 struct ext4_fc_del_range lrange;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001739 struct ext4_map_blocks map;
1740 ext4_lblk_t cur, remaining;
1741 int ret;
1742
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001743 memcpy(&lrange, val, sizeof(lrange));
1744 cur = le32_to_cpu(lrange.fc_lblk);
1745 remaining = le32_to_cpu(lrange.fc_len);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001746
1747 trace_ext4_fc_replay(sb, EXT4_FC_TAG_DEL_RANGE,
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001748 le32_to_cpu(lrange.fc_ino), cur, remaining);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001749
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001750 inode = ext4_iget(sb, le32_to_cpu(lrange.fc_ino), EXT4_IGET_NORMAL);
Yi Li6c557cb12020-12-30 11:38:27 +08001751 if (IS_ERR(inode)) {
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001752 jbd_debug(1, "Inode %d not found", le32_to_cpu(lrange.fc_ino));
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001753 return 0;
1754 }
1755
1756 ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301757 if (ret)
1758 goto out;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001759
1760 jbd_debug(1, "DEL_RANGE, inode %ld, lblk %d, len %d\n",
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001761 inode->i_ino, le32_to_cpu(lrange.fc_lblk),
1762 le32_to_cpu(lrange.fc_len));
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001763 while (remaining > 0) {
1764 map.m_lblk = cur;
1765 map.m_len = remaining;
1766
1767 ret = ext4_map_blocks(NULL, inode, &map, 0);
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301768 if (ret < 0)
1769 goto out;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001770 if (ret > 0) {
1771 remaining -= ret;
1772 cur += ret;
1773 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0);
1774 } else {
1775 remaining -= map.m_len;
1776 cur += map.m_len;
1777 }
1778 }
1779
Xin Yin04b56272021-12-23 11:23:36 +08001780 down_write(&EXT4_I(inode)->i_data_sem);
Xin Yin84b76a52022-01-26 14:31:46 +08001781 ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk),
1782 le32_to_cpu(lrange.fc_lblk) +
1783 le32_to_cpu(lrange.fc_len) - 1);
Xin Yin04b56272021-12-23 11:23:36 +08001784 up_write(&EXT4_I(inode)->i_data_sem);
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301785 if (ret)
1786 goto out;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001787 ext4_ext_replay_shrink_inode(inode,
1788 i_size_read(inode) >> sb->s_blocksize_bits);
1789 ext4_mark_inode_dirty(NULL, inode);
Ritesh Harjani62e46e02022-01-17 17:41:49 +05301790out:
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001791 iput(inode);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001792 return 0;
1793}
1794
1795static inline const char *tag2str(u16 tag)
1796{
1797 switch (tag) {
1798 case EXT4_FC_TAG_LINK:
1799 return "TAG_ADD_ENTRY";
1800 case EXT4_FC_TAG_UNLINK:
1801 return "TAG_DEL_ENTRY";
1802 case EXT4_FC_TAG_ADD_RANGE:
1803 return "TAG_ADD_RANGE";
1804 case EXT4_FC_TAG_CREAT:
1805 return "TAG_CREAT_DENTRY";
1806 case EXT4_FC_TAG_DEL_RANGE:
1807 return "TAG_DEL_RANGE";
1808 case EXT4_FC_TAG_INODE:
1809 return "TAG_INODE";
1810 case EXT4_FC_TAG_PAD:
1811 return "TAG_PAD";
1812 case EXT4_FC_TAG_TAIL:
1813 return "TAG_TAIL";
1814 case EXT4_FC_TAG_HEAD:
1815 return "TAG_HEAD";
1816 default:
1817 return "TAG_ERROR";
1818 }
1819}
1820
1821static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb)
1822{
1823 struct ext4_fc_replay_state *state;
1824 struct inode *inode;
1825 struct ext4_ext_path *path = NULL;
1826 struct ext4_map_blocks map;
1827 int i, ret, j;
1828 ext4_lblk_t cur, end;
1829
1830 state = &EXT4_SB(sb)->s_fc_replay_state;
1831 for (i = 0; i < state->fc_modified_inodes_used; i++) {
1832 inode = ext4_iget(sb, state->fc_modified_inodes[i],
1833 EXT4_IGET_NORMAL);
Yi Li6c557cb12020-12-30 11:38:27 +08001834 if (IS_ERR(inode)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001835 jbd_debug(1, "Inode %d not found.",
1836 state->fc_modified_inodes[i]);
1837 continue;
1838 }
1839 cur = 0;
1840 end = EXT_MAX_BLOCKS;
1841 while (cur < end) {
1842 map.m_lblk = cur;
1843 map.m_len = end - cur;
1844
1845 ret = ext4_map_blocks(NULL, inode, &map, 0);
1846 if (ret < 0)
1847 break;
1848
1849 if (ret > 0) {
1850 path = ext4_find_extent(inode, map.m_lblk, NULL, 0);
Yi Li6c557cb12020-12-30 11:38:27 +08001851 if (!IS_ERR(path)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001852 for (j = 0; j < path->p_depth; j++)
1853 ext4_mb_mark_bb(inode->i_sb,
1854 path[j].p_block, 1, 1);
1855 ext4_ext_drop_refs(path);
1856 kfree(path);
1857 }
1858 cur += ret;
1859 ext4_mb_mark_bb(inode->i_sb, map.m_pblk,
1860 map.m_len, 1);
1861 } else {
1862 cur = cur + (map.m_len ? map.m_len : 1);
1863 }
1864 }
1865 iput(inode);
1866 }
1867}
1868
1869/*
1870 * Check if block is in excluded regions for block allocation. The simple
1871 * allocator that runs during replay phase is calls this function to see
1872 * if it is okay to use a block.
1873 */
1874bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t blk)
1875{
1876 int i;
1877 struct ext4_fc_replay_state *state;
1878
1879 state = &EXT4_SB(sb)->s_fc_replay_state;
1880 for (i = 0; i < state->fc_regions_valid; i++) {
1881 if (state->fc_regions[i].ino == 0 ||
1882 state->fc_regions[i].len == 0)
1883 continue;
1884 if (blk >= state->fc_regions[i].pblk &&
1885 blk < state->fc_regions[i].pblk + state->fc_regions[i].len)
1886 return true;
1887 }
1888 return false;
1889}
1890
1891/* Cleanup function called after replay */
1892void ext4_fc_replay_cleanup(struct super_block *sb)
1893{
1894 struct ext4_sb_info *sbi = EXT4_SB(sb);
1895
1896 sbi->s_mount_state &= ~EXT4_FC_REPLAY;
1897 kfree(sbi->s_fc_replay_state.fc_regions);
1898 kfree(sbi->s_fc_replay_state.fc_modified_inodes);
1899}
1900
1901/*
1902 * Recovery Scan phase handler
1903 *
1904 * This function is called during the scan phase and is responsible
1905 * for doing following things:
1906 * - Make sure the fast commit area has valid tags for replay
1907 * - Count number of tags that need to be replayed by the replay handler
1908 * - Verify CRC
1909 * - Create a list of excluded blocks for allocation during replay phase
1910 *
1911 * This function returns JBD2_FC_REPLAY_CONTINUE to indicate that SCAN is
1912 * incomplete and JBD2 should send more blocks. It returns JBD2_FC_REPLAY_STOP
1913 * to indicate that scan has finished and JBD2 can now start replay phase.
1914 * It returns a negative error to indicate that there was an error. At the end
1915 * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set
1916 * to indicate the number of tags that need to replayed during the replay phase.
1917 */
1918static int ext4_fc_replay_scan(journal_t *journal,
1919 struct buffer_head *bh, int off,
1920 tid_t expected_tid)
1921{
1922 struct super_block *sb = journal->j_private;
1923 struct ext4_sb_info *sbi = EXT4_SB(sb);
1924 struct ext4_fc_replay_state *state;
1925 int ret = JBD2_FC_REPLAY_CONTINUE;
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001926 struct ext4_fc_add_range ext;
1927 struct ext4_fc_tl tl;
1928 struct ext4_fc_tail tail;
1929 __u8 *start, *end, *cur, *val;
1930 struct ext4_fc_head head;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001931 struct ext4_extent *ex;
1932
1933 state = &sbi->s_fc_replay_state;
1934
1935 start = (u8 *)bh->b_data;
1936 end = (__u8 *)bh->b_data + journal->j_blocksize - 1;
1937
1938 if (state->fc_replay_expected_off == 0) {
1939 state->fc_cur_tag = 0;
1940 state->fc_replay_num_tags = 0;
1941 state->fc_crc = 0;
1942 state->fc_regions = NULL;
1943 state->fc_regions_valid = state->fc_regions_used =
1944 state->fc_regions_size = 0;
1945 /* Check if we can stop early */
1946 if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag)
1947 != EXT4_FC_TAG_HEAD)
1948 return 0;
1949 }
1950
1951 if (off != state->fc_replay_expected_off) {
1952 ret = -EFSCORRUPTED;
1953 goto out_err;
1954 }
1955
1956 state->fc_replay_expected_off++;
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001957 for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) {
1958 memcpy(&tl, cur, sizeof(tl));
1959 val = cur + sizeof(tl);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001960 jbd_debug(3, "Scan phase, tag:%s, blk %lld\n",
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001961 tag2str(le16_to_cpu(tl.fc_tag)), bh->b_blocknr);
1962 switch (le16_to_cpu(tl.fc_tag)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001963 case EXT4_FC_TAG_ADD_RANGE:
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001964 memcpy(&ext, val, sizeof(ext));
1965 ex = (struct ext4_extent *)&ext.fc_ex;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001966 ret = ext4_fc_record_regions(sb,
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001967 le32_to_cpu(ext.fc_ino),
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001968 le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex),
Xin Yin8d71fc22022-01-10 11:51:40 +08001969 ext4_ext_get_actual_len(ex), 0);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001970 if (ret < 0)
1971 break;
1972 ret = JBD2_FC_REPLAY_CONTINUE;
1973 fallthrough;
1974 case EXT4_FC_TAG_DEL_RANGE:
1975 case EXT4_FC_TAG_LINK:
1976 case EXT4_FC_TAG_UNLINK:
1977 case EXT4_FC_TAG_CREAT:
1978 case EXT4_FC_TAG_INODE:
1979 case EXT4_FC_TAG_PAD:
1980 state->fc_cur_tag++;
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001981 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
1982 sizeof(tl) + le16_to_cpu(tl.fc_len));
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001983 break;
1984 case EXT4_FC_TAG_TAIL:
1985 state->fc_cur_tag++;
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001986 memcpy(&tail, val, sizeof(tail));
1987 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
1988 sizeof(tl) +
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001989 offsetof(struct ext4_fc_tail,
1990 fc_crc));
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07001991 if (le32_to_cpu(tail.fc_tid) == expected_tid &&
1992 le32_to_cpu(tail.fc_crc) == state->fc_crc) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07001993 state->fc_replay_num_tags = state->fc_cur_tag;
1994 state->fc_regions_valid =
1995 state->fc_regions_used;
1996 } else {
1997 ret = state->fc_replay_num_tags ?
1998 JBD2_FC_REPLAY_STOP : -EFSBADCRC;
1999 }
2000 state->fc_crc = 0;
2001 break;
2002 case EXT4_FC_TAG_HEAD:
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002003 memcpy(&head, val, sizeof(head));
2004 if (le32_to_cpu(head.fc_features) &
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002005 ~EXT4_FC_SUPPORTED_FEATURES) {
2006 ret = -EOPNOTSUPP;
2007 break;
2008 }
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002009 if (le32_to_cpu(head.fc_tid) != expected_tid) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002010 ret = JBD2_FC_REPLAY_STOP;
2011 break;
2012 }
2013 state->fc_cur_tag++;
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002014 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2015 sizeof(tl) + le16_to_cpu(tl.fc_len));
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002016 break;
2017 default:
2018 ret = state->fc_replay_num_tags ?
2019 JBD2_FC_REPLAY_STOP : -ECANCELED;
2020 }
2021 if (ret < 0 || ret == JBD2_FC_REPLAY_STOP)
2022 break;
2023 }
2024
2025out_err:
2026 trace_ext4_fc_replay_scan(sb, ret, off);
2027 return ret;
2028}
2029
Harshad Shirwadkar5b849b52020-10-15 13:37:58 -07002030/*
2031 * Main recovery path entry point.
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002032 * The meaning of return codes is similar as above.
Harshad Shirwadkar5b849b52020-10-15 13:37:58 -07002033 */
2034static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh,
2035 enum passtype pass, int off, tid_t expected_tid)
2036{
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002037 struct super_block *sb = journal->j_private;
2038 struct ext4_sb_info *sbi = EXT4_SB(sb);
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002039 struct ext4_fc_tl tl;
2040 __u8 *start, *end, *cur, *val;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002041 int ret = JBD2_FC_REPLAY_CONTINUE;
2042 struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state;
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002043 struct ext4_fc_tail tail;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002044
2045 if (pass == PASS_SCAN) {
2046 state->fc_current_pass = PASS_SCAN;
2047 return ext4_fc_replay_scan(journal, bh, off, expected_tid);
2048 }
2049
2050 if (state->fc_current_pass != pass) {
2051 state->fc_current_pass = pass;
2052 sbi->s_mount_state |= EXT4_FC_REPLAY;
2053 }
2054 if (!sbi->s_fc_replay_state.fc_replay_num_tags) {
2055 jbd_debug(1, "Replay stops\n");
2056 ext4_fc_set_bitmaps_and_counters(sb);
2057 return 0;
2058 }
2059
2060#ifdef CONFIG_EXT4_DEBUG
2061 if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) {
2062 pr_warn("Dropping fc block %d because max_replay set\n", off);
2063 return JBD2_FC_REPLAY_STOP;
2064 }
2065#endif
2066
2067 start = (u8 *)bh->b_data;
2068 end = (__u8 *)bh->b_data + journal->j_blocksize - 1;
2069
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002070 for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) {
2071 memcpy(&tl, cur, sizeof(tl));
2072 val = cur + sizeof(tl);
2073
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002074 if (state->fc_replay_num_tags == 0) {
2075 ret = JBD2_FC_REPLAY_STOP;
2076 ext4_fc_set_bitmaps_and_counters(sb);
2077 break;
2078 }
2079 jbd_debug(3, "Replay phase, tag:%s\n",
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002080 tag2str(le16_to_cpu(tl.fc_tag)));
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002081 state->fc_replay_num_tags--;
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002082 switch (le16_to_cpu(tl.fc_tag)) {
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002083 case EXT4_FC_TAG_LINK:
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002084 ret = ext4_fc_replay_link(sb, &tl, val);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002085 break;
2086 case EXT4_FC_TAG_UNLINK:
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002087 ret = ext4_fc_replay_unlink(sb, &tl, val);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002088 break;
2089 case EXT4_FC_TAG_ADD_RANGE:
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002090 ret = ext4_fc_replay_add_range(sb, &tl, val);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002091 break;
2092 case EXT4_FC_TAG_CREAT:
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002093 ret = ext4_fc_replay_create(sb, &tl, val);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002094 break;
2095 case EXT4_FC_TAG_DEL_RANGE:
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002096 ret = ext4_fc_replay_del_range(sb, &tl, val);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002097 break;
2098 case EXT4_FC_TAG_INODE:
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002099 ret = ext4_fc_replay_inode(sb, &tl, val);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002100 break;
2101 case EXT4_FC_TAG_PAD:
2102 trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0,
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002103 le16_to_cpu(tl.fc_len), 0);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002104 break;
2105 case EXT4_FC_TAG_TAIL:
2106 trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, 0,
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002107 le16_to_cpu(tl.fc_len), 0);
2108 memcpy(&tail, val, sizeof(tail));
2109 WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002110 break;
2111 case EXT4_FC_TAG_HEAD:
2112 break;
2113 default:
Harshad Shirwadkarfb86acc2021-05-19 14:59:20 -07002114 trace_ext4_fc_replay(sb, le16_to_cpu(tl.fc_tag), 0,
2115 le16_to_cpu(tl.fc_len), 0);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07002116 ret = -ECANCELED;
2117 break;
2118 }
2119 if (ret < 0)
2120 break;
2121 ret = JBD2_FC_REPLAY_CONTINUE;
2122 }
2123 return ret;
Harshad Shirwadkar5b849b52020-10-15 13:37:58 -07002124}
2125
Harshad Shirwadkar6866d7b2020-10-15 13:37:55 -07002126void ext4_fc_init(struct super_block *sb, journal_t *journal)
2127{
Harshad Shirwadkar5b849b52020-10-15 13:37:58 -07002128 /*
2129 * We set replay callback even if fast commit disabled because we may
2130 * could still have fast commit blocks that need to be replayed even if
2131 * fast commit has now been turned off.
2132 */
2133 journal->j_fc_replay_callback = ext4_fc_replay;
Harshad Shirwadkar6866d7b2020-10-15 13:37:55 -07002134 if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
2135 return;
Harshad Shirwadkarff780b92020-10-15 13:37:56 -07002136 journal->j_fc_cleanup_callback = ext4_fc_cleanup;
Harshad Shirwadkar6866d7b2020-10-15 13:37:55 -07002137}
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07002138
Theodore Ts'ofa329e2732020-11-06 23:59:42 -05002139static const char *fc_ineligible_reasons[] = {
Harshad Shirwadkarce8c59d2020-10-15 13:38:01 -07002140 "Extended attributes changed",
2141 "Cross rename",
2142 "Journal flag changed",
2143 "Insufficient memory",
2144 "Swap boot",
2145 "Resize",
2146 "Dir renamed",
2147 "Falloc range op",
Harshad Shirwadkar556e0312020-11-05 19:59:07 -08002148 "Data journalling",
Harshad Shirwadkarce8c59d2020-10-15 13:38:01 -07002149 "FC Commit Failed"
2150};
2151
2152int ext4_fc_info_show(struct seq_file *seq, void *v)
2153{
2154 struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private);
2155 struct ext4_fc_stats *stats = &sbi->s_fc_stats;
2156 int i;
2157
2158 if (v != SEQ_START_TOKEN)
2159 return 0;
2160
2161 seq_printf(seq,
2162 "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n",
2163 stats->fc_num_commits, stats->fc_ineligible_commits,
2164 stats->fc_numblks,
2165 div_u64(sbi->s_fc_avg_commit_time, 1000));
2166 seq_puts(seq, "Ineligible reasons:\n");
2167 for (i = 0; i < EXT4_FC_REASON_MAX; i++)
2168 seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i],
2169 stats->fc_ineligible_reason_count[i]);
2170
2171 return 0;
2172}
2173
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07002174int __init ext4_fc_init_dentry_cache(void)
2175{
2176 ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update,
2177 SLAB_RECLAIM_ACCOUNT);
2178
2179 if (ext4_fc_dentry_cachep == NULL)
2180 return -ENOMEM;
2181
2182 return 0;
2183}
Sebastian Andrzej Siewiord60e9da2021-12-23 17:44:36 +01002184
2185void ext4_fc_destroy_dentry_cache(void)
2186{
2187 kmem_cache_destroy(ext4_fc_dentry_cachep);
2188}