blob: bae409aad53ec8006563cf21111a59b439a097db [file] [log] [blame]
Theodore Ts'of5166762017-12-17 22:00:59 -05001// SPDX-License-Identifier: GPL-2.0
Alex Tomasa86c6182006-10-11 01:21:03 -07002/*
3 * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
4 * Written by Alex Tomas <alex@clusterfs.com>
5 *
6 * Architecture independence:
7 * Copyright (c) 2005, Bull S.A.
8 * Written by Pierre Peiffer <pierre.peiffer@bull.net>
Alex Tomasa86c6182006-10-11 01:21:03 -07009 */
10
11/*
12 * Extents support for EXT4
13 *
14 * TODO:
15 * - ext4*_error() should be used in some situations
16 * - analyze all BUG()/BUG_ON(), use -EIO where appropriate
17 * - smart tree reduction
18 */
19
Alex Tomasa86c6182006-10-11 01:21:03 -070020#include <linux/fs.h>
21#include <linux/time.h>
Mingming Caocd02ff02007-10-16 18:38:25 -040022#include <linux/jbd2.h>
Alex Tomasa86c6182006-10-11 01:21:03 -070023#include <linux/highuid.h>
24#include <linux/pagemap.h>
25#include <linux/quotaops.h>
26#include <linux/string.h>
27#include <linux/slab.h>
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080028#include <linux/uaccess.h>
Eric Sandeen6873fa02008-10-07 00:46:36 -040029#include <linux/fiemap.h>
Tejun Heo66114ca2015-05-22 17:13:32 -040030#include <linux/backing-dev.h>
Christoph Hellwig3dcf5452008-04-29 18:13:32 -040031#include "ext4_jbd2.h"
Theodore Ts'o4a092d72012-11-28 13:03:30 -050032#include "ext4_extents.h"
Tao Maf19d5872012-12-10 14:05:51 -050033#include "xattr.h"
Alex Tomasa86c6182006-10-11 01:21:03 -070034
Jiaying Zhang0562e0b2011-03-21 21:38:05 -040035#include <trace/events/ext4.h>
36
Lukas Czerner5f95d212012-03-19 23:03:19 -040037/*
38 * used by extent splitting.
39 */
40#define EXT4_EXT_MAY_ZEROOUT 0x1 /* safe to zeroout if split fails \
41 due to ENOSPC */
Lukas Czerner556615d2014-04-20 23:45:47 -040042#define EXT4_EXT_MARK_UNWRIT1 0x2 /* mark first half unwritten */
43#define EXT4_EXT_MARK_UNWRIT2 0x4 /* mark second half unwritten */
Lukas Czerner5f95d212012-03-19 23:03:19 -040044
Dmitry Monakhovdee1f972012-10-10 01:04:58 -040045#define EXT4_EXT_DATA_VALID1 0x8 /* first half contains valid data */
46#define EXT4_EXT_DATA_VALID2 0x10 /* second half contains valid data */
47
Darrick J. Wong7ac59902012-04-29 18:37:10 -040048static __le32 ext4_extent_block_csum(struct inode *inode,
49 struct ext4_extent_header *eh)
50{
51 struct ext4_inode_info *ei = EXT4_I(inode);
52 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
53 __u32 csum;
54
55 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)eh,
56 EXT4_EXTENT_TAIL_OFFSET(eh));
57 return cpu_to_le32(csum);
58}
59
60static int ext4_extent_block_csum_verify(struct inode *inode,
61 struct ext4_extent_header *eh)
62{
63 struct ext4_extent_tail *et;
64
Dmitry Monakhov9aa5d32b2014-10-13 03:36:16 -040065 if (!ext4_has_metadata_csum(inode->i_sb))
Darrick J. Wong7ac59902012-04-29 18:37:10 -040066 return 1;
67
68 et = find_ext4_extent_tail(eh);
69 if (et->et_checksum != ext4_extent_block_csum(inode, eh))
70 return 0;
71 return 1;
72}
73
74static void ext4_extent_block_csum_set(struct inode *inode,
75 struct ext4_extent_header *eh)
76{
77 struct ext4_extent_tail *et;
78
Dmitry Monakhov9aa5d32b2014-10-13 03:36:16 -040079 if (!ext4_has_metadata_csum(inode->i_sb))
Darrick J. Wong7ac59902012-04-29 18:37:10 -040080 return;
81
82 et = find_ext4_extent_tail(eh);
83 et->et_checksum = ext4_extent_block_csum(inode, eh);
84}
85
Allison Hendersond583fb82011-05-25 07:41:43 -040086static int ext4_split_extent(handle_t *handle,
87 struct inode *inode,
Theodore Ts'odfe50802014-09-01 14:37:09 -040088 struct ext4_ext_path **ppath,
Allison Hendersond583fb82011-05-25 07:41:43 -040089 struct ext4_map_blocks *map,
90 int split_flag,
91 int flags);
92
Lukas Czerner5f95d212012-03-19 23:03:19 -040093static int ext4_split_extent_at(handle_t *handle,
94 struct inode *inode,
Theodore Ts'odfe50802014-09-01 14:37:09 -040095 struct ext4_ext_path **ppath,
Lukas Czerner5f95d212012-03-19 23:03:19 -040096 ext4_lblk_t split,
97 int split_flag,
98 int flags);
99
Lukas Czerner91dd8c12012-11-28 12:32:26 -0500100static int ext4_find_delayed_extent(struct inode *inode,
Zheng Liu69eb33d2013-02-18 00:31:07 -0500101 struct extent_status *newes);
Lukas Czerner91dd8c12012-11-28 12:32:26 -0500102
Jan Karaa4130362019-11-05 17:44:16 +0100103static int ext4_ext_trunc_restart_fn(struct inode *inode, int *dropped)
Alex Tomasa86c6182006-10-11 01:21:03 -0700104{
Theodore Ts'o7b808192016-04-25 23:13:17 -0400105 /*
Jan Karaa4130362019-11-05 17:44:16 +0100106 * Drop i_data_sem to avoid deadlock with ext4_map_blocks. At this
107 * moment, get_block can be called only for blocks inside i_size since
108 * page cache has been already dropped and writes are blocked by
109 * i_mutex. So we can safely drop the i_data_sem here.
Theodore Ts'o7b808192016-04-25 23:13:17 -0400110 */
Jan Karaa4130362019-11-05 17:44:16 +0100111 BUG_ON(EXT4_JOURNAL(inode) == NULL);
112 ext4_discard_preallocations(inode);
113 up_write(&EXT4_I(inode)->i_data_sem);
114 *dropped = 1;
115 return 0;
116}
Jan Kara487caee2009-08-17 22:17:20 -0400117
Jan Karaa4130362019-11-05 17:44:16 +0100118/*
119 * Make sure 'handle' has at least 'check_cred' credits. If not, restart
120 * transaction with 'restart_cred' credits. The function drops i_data_sem
121 * when restarting transaction and gets it after transaction is restarted.
122 *
123 * The function returns 0 on success, 1 if transaction had to be restarted,
124 * and < 0 in case of fatal error.
125 */
126int ext4_datasem_ensure_credits(handle_t *handle, struct inode *inode,
Jan Kara83448bd2019-11-05 17:44:29 +0100127 int check_cred, int restart_cred,
128 int revoke_cred)
Jan Karaa4130362019-11-05 17:44:16 +0100129{
130 int ret;
131 int dropped = 0;
132
133 ret = ext4_journal_ensure_credits_fn(handle, check_cred, restart_cred,
Jan Kara83448bd2019-11-05 17:44:29 +0100134 revoke_cred, ext4_ext_trunc_restart_fn(inode, &dropped));
Jan Karaa4130362019-11-05 17:44:16 +0100135 if (dropped)
136 down_write(&EXT4_I(inode)->i_data_sem);
137 return ret;
Alex Tomasa86c6182006-10-11 01:21:03 -0700138}
139
140/*
141 * could return:
142 * - EROFS
143 * - ENOMEM
144 */
145static int ext4_ext_get_access(handle_t *handle, struct inode *inode,
146 struct ext4_ext_path *path)
147{
148 if (path->p_bh) {
149 /* path points to block */
liang xie5d601252014-05-12 22:06:43 -0400150 BUFFER_TRACE(path->p_bh, "get_write_access");
Alex Tomasa86c6182006-10-11 01:21:03 -0700151 return ext4_journal_get_write_access(handle, path->p_bh);
152 }
153 /* path points to leaf/index in inode body */
154 /* we use in-core data, no need to protect them */
155 return 0;
156}
157
158/*
159 * could return:
160 * - EROFS
161 * - ENOMEM
162 * - EIO
163 */
Eric Biggers43f81672019-12-31 12:04:40 -0600164static int __ext4_ext_dirty(const char *where, unsigned int line,
165 handle_t *handle, struct inode *inode,
166 struct ext4_ext_path *path)
Alex Tomasa86c6182006-10-11 01:21:03 -0700167{
168 int err;
Dmitry Monakhov4b1f1662014-07-27 22:28:15 -0400169
170 WARN_ON(!rwsem_is_locked(&EXT4_I(inode)->i_data_sem));
Alex Tomasa86c6182006-10-11 01:21:03 -0700171 if (path->p_bh) {
Darrick J. Wong7ac59902012-04-29 18:37:10 -0400172 ext4_extent_block_csum_set(inode, ext_block_hdr(path->p_bh));
Alex Tomasa86c6182006-10-11 01:21:03 -0700173 /* path points to block */
Theodore Ts'o9ea7a0d2011-09-04 10:18:14 -0400174 err = __ext4_handle_dirty_metadata(where, line, handle,
175 inode, path->p_bh);
Alex Tomasa86c6182006-10-11 01:21:03 -0700176 } else {
177 /* path points to leaf/index in inode body */
178 err = ext4_mark_inode_dirty(handle, inode);
179 }
180 return err;
181}
182
Eric Biggers43f81672019-12-31 12:04:40 -0600183#define ext4_ext_dirty(handle, inode, path) \
184 __ext4_ext_dirty(__func__, __LINE__, (handle), (inode), (path))
185
Alex Tomasf65e6fb2006-10-11 01:21:05 -0700186static ext4_fsblk_t ext4_ext_find_goal(struct inode *inode,
Alex Tomasa86c6182006-10-11 01:21:03 -0700187 struct ext4_ext_path *path,
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -0500188 ext4_lblk_t block)
Alex Tomasa86c6182006-10-11 01:21:03 -0700189{
Alex Tomasa86c6182006-10-11 01:21:03 -0700190 if (path) {
Yongqiang Yang81fdbb42011-10-29 09:23:38 -0400191 int depth = path->p_depth;
Alex Tomasa86c6182006-10-11 01:21:03 -0700192 struct ext4_extent *ex;
Alex Tomasa86c6182006-10-11 01:21:03 -0700193
Kazuya Mioad4fb9c2011-01-10 12:12:28 -0500194 /*
195 * Try to predict block placement assuming that we are
196 * filling in a file which will eventually be
197 * non-sparse --- i.e., in the case of libbfd writing
198 * an ELF object sections out-of-order but in a way
199 * the eventually results in a contiguous object or
200 * executable file, or some database extending a table
201 * space file. However, this is actually somewhat
202 * non-ideal if we are writing a sparse file such as
203 * qemu or KVM writing a raw image file that is going
204 * to stay fairly sparse, since it will end up
205 * fragmenting the file system's free space. Maybe we
206 * should have some hueristics or some way to allow
207 * userspace to pass a hint to file system,
Tao Mab8d65682011-01-21 23:21:31 +0800208 * especially if the latter case turns out to be
Kazuya Mioad4fb9c2011-01-10 12:12:28 -0500209 * common.
210 */
Avantika Mathur7e028972006-12-06 20:41:33 -0800211 ex = path[depth].p_ext;
Kazuya Mioad4fb9c2011-01-10 12:12:28 -0500212 if (ex) {
213 ext4_fsblk_t ext_pblk = ext4_ext_pblock(ex);
214 ext4_lblk_t ext_block = le32_to_cpu(ex->ee_block);
215
216 if (block > ext_block)
217 return ext_pblk + (block - ext_block);
218 else
219 return ext_pblk - (ext_block - block);
220 }
Alex Tomasa86c6182006-10-11 01:21:03 -0700221
Randy Dunlapd0d856e2006-10-11 01:21:07 -0700222 /* it looks like index is empty;
223 * try to find starting block from index itself */
Alex Tomasa86c6182006-10-11 01:21:03 -0700224 if (path[depth].p_bh)
225 return path[depth].p_bh->b_blocknr;
226 }
227
228 /* OK. use inode's group */
Eric Sandeenf86186b2011-06-28 10:01:31 -0400229 return ext4_inode_to_goal_block(inode);
Alex Tomasa86c6182006-10-11 01:21:03 -0700230}
231
Aneesh Kumar K.V654b4902008-07-11 19:27:31 -0400232/*
233 * Allocation for a meta data block
234 */
Alex Tomasf65e6fb2006-10-11 01:21:05 -0700235static ext4_fsblk_t
Aneesh Kumar K.V654b4902008-07-11 19:27:31 -0400236ext4_ext_new_meta_block(handle_t *handle, struct inode *inode,
Alex Tomasa86c6182006-10-11 01:21:03 -0700237 struct ext4_ext_path *path,
Allison Henderson55f020d2011-05-25 07:41:26 -0400238 struct ext4_extent *ex, int *err, unsigned int flags)
Alex Tomasa86c6182006-10-11 01:21:03 -0700239{
Alex Tomasf65e6fb2006-10-11 01:21:05 -0700240 ext4_fsblk_t goal, newblock;
Alex Tomasa86c6182006-10-11 01:21:03 -0700241
242 goal = ext4_ext_find_goal(inode, path, le32_to_cpu(ex->ee_block));
Allison Henderson55f020d2011-05-25 07:41:26 -0400243 newblock = ext4_new_meta_blocks(handle, inode, goal, flags,
244 NULL, err);
Alex Tomasa86c6182006-10-11 01:21:03 -0700245 return newblock;
246}
247
Theodore Ts'o55ad63b2009-08-28 10:40:33 -0400248static inline int ext4_ext_space_block(struct inode *inode, int check)
Alex Tomasa86c6182006-10-11 01:21:03 -0700249{
250 int size;
251
252 size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header))
253 / sizeof(struct ext4_extent);
Robert P. J. Daybbf2f9f2007-02-17 19:20:16 +0100254#ifdef AGGRESSIVE_TEST
Yongqiang Yang02dc62fb2011-10-29 09:29:11 -0400255 if (!check && size > 6)
256 size = 6;
Alex Tomasa86c6182006-10-11 01:21:03 -0700257#endif
258 return size;
259}
260
Theodore Ts'o55ad63b2009-08-28 10:40:33 -0400261static inline int ext4_ext_space_block_idx(struct inode *inode, int check)
Alex Tomasa86c6182006-10-11 01:21:03 -0700262{
263 int size;
264
265 size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header))
266 / sizeof(struct ext4_extent_idx);
Robert P. J. Daybbf2f9f2007-02-17 19:20:16 +0100267#ifdef AGGRESSIVE_TEST
Yongqiang Yang02dc62fb2011-10-29 09:29:11 -0400268 if (!check && size > 5)
269 size = 5;
Alex Tomasa86c6182006-10-11 01:21:03 -0700270#endif
271 return size;
272}
273
Theodore Ts'o55ad63b2009-08-28 10:40:33 -0400274static inline int ext4_ext_space_root(struct inode *inode, int check)
Alex Tomasa86c6182006-10-11 01:21:03 -0700275{
276 int size;
277
278 size = sizeof(EXT4_I(inode)->i_data);
279 size -= sizeof(struct ext4_extent_header);
280 size /= sizeof(struct ext4_extent);
Robert P. J. Daybbf2f9f2007-02-17 19:20:16 +0100281#ifdef AGGRESSIVE_TEST
Yongqiang Yang02dc62fb2011-10-29 09:29:11 -0400282 if (!check && size > 3)
283 size = 3;
Alex Tomasa86c6182006-10-11 01:21:03 -0700284#endif
285 return size;
286}
287
Theodore Ts'o55ad63b2009-08-28 10:40:33 -0400288static inline int ext4_ext_space_root_idx(struct inode *inode, int check)
Alex Tomasa86c6182006-10-11 01:21:03 -0700289{
290 int size;
291
292 size = sizeof(EXT4_I(inode)->i_data);
293 size -= sizeof(struct ext4_extent_header);
294 size /= sizeof(struct ext4_extent_idx);
Robert P. J. Daybbf2f9f2007-02-17 19:20:16 +0100295#ifdef AGGRESSIVE_TEST
Yongqiang Yang02dc62fb2011-10-29 09:29:11 -0400296 if (!check && size > 4)
297 size = 4;
Alex Tomasa86c6182006-10-11 01:21:03 -0700298#endif
299 return size;
300}
301
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -0400302static inline int
303ext4_force_split_extent_at(handle_t *handle, struct inode *inode,
Theodore Ts'odfe50802014-09-01 14:37:09 -0400304 struct ext4_ext_path **ppath, ext4_lblk_t lblk,
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -0400305 int nofail)
306{
Theodore Ts'odfe50802014-09-01 14:37:09 -0400307 struct ext4_ext_path *path = *ppath;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -0400308 int unwritten = ext4_ext_is_unwritten(path[path->p_depth].p_ext);
309
Theodore Ts'odfe50802014-09-01 14:37:09 -0400310 return ext4_split_extent_at(handle, inode, ppath, lblk, unwritten ?
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -0400311 EXT4_EXT_MARK_UNWRIT1|EXT4_EXT_MARK_UNWRIT2 : 0,
312 EXT4_EX_NOCACHE | EXT4_GET_BLOCKS_PRE_IO |
313 (nofail ? EXT4_GET_BLOCKS_METADATA_NOFAIL:0));
314}
315
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400316static int
317ext4_ext_max_entries(struct inode *inode, int depth)
318{
319 int max;
320
321 if (depth == ext_depth(inode)) {
322 if (depth == 0)
Theodore Ts'o55ad63b2009-08-28 10:40:33 -0400323 max = ext4_ext_space_root(inode, 1);
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400324 else
Theodore Ts'o55ad63b2009-08-28 10:40:33 -0400325 max = ext4_ext_space_root_idx(inode, 1);
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400326 } else {
327 if (depth == 0)
Theodore Ts'o55ad63b2009-08-28 10:40:33 -0400328 max = ext4_ext_space_block(inode, 1);
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400329 else
Theodore Ts'o55ad63b2009-08-28 10:40:33 -0400330 max = ext4_ext_space_block_idx(inode, 1);
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400331 }
332
333 return max;
334}
335
Aneesh Kumar K.V56b19862009-03-12 09:51:20 -0400336static int ext4_valid_extent(struct inode *inode, struct ext4_extent *ext)
337{
Theodore Ts'obf89d162010-10-27 21:30:14 -0400338 ext4_fsblk_t block = ext4_ext_pblock(ext);
Aneesh Kumar K.V56b19862009-03-12 09:51:20 -0400339 int len = ext4_ext_get_actual_len(ext);
Eryu Guan5946d082013-12-03 21:22:21 -0500340 ext4_lblk_t lblock = le32_to_cpu(ext->ee_block);
Theodore Ts'oe84a26c2009-04-22 20:52:25 -0400341
Vegard Nossumf70749c2016-06-30 11:53:46 -0400342 /*
343 * We allow neither:
344 * - zero length
345 * - overflow/wrap-around
346 */
347 if (lblock + len <= lblock)
Theodore Ts'o31d4f3a2012-03-11 23:30:16 -0400348 return 0;
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400349 return ext4_data_block_valid(EXT4_SB(inode->i_sb), block, len);
Aneesh Kumar K.V56b19862009-03-12 09:51:20 -0400350}
351
352static int ext4_valid_extent_idx(struct inode *inode,
353 struct ext4_extent_idx *ext_idx)
354{
Theodore Ts'obf89d162010-10-27 21:30:14 -0400355 ext4_fsblk_t block = ext4_idx_pblock(ext_idx);
Theodore Ts'oe84a26c2009-04-22 20:52:25 -0400356
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400357 return ext4_data_block_valid(EXT4_SB(inode->i_sb), block, 1);
Aneesh Kumar K.V56b19862009-03-12 09:51:20 -0400358}
359
360static int ext4_valid_extent_entries(struct inode *inode,
361 struct ext4_extent_header *eh,
362 int depth)
363{
Aneesh Kumar K.V56b19862009-03-12 09:51:20 -0400364 unsigned short entries;
365 if (eh->eh_entries == 0)
366 return 1;
367
368 entries = le16_to_cpu(eh->eh_entries);
369
370 if (depth == 0) {
371 /* leaf entries */
Yongqiang Yang81fdbb42011-10-29 09:23:38 -0400372 struct ext4_extent *ext = EXT_FIRST_EXTENT(eh);
Eryu Guan5946d082013-12-03 21:22:21 -0500373 struct ext4_super_block *es = EXT4_SB(inode->i_sb)->s_es;
374 ext4_fsblk_t pblock = 0;
375 ext4_lblk_t lblock = 0;
376 ext4_lblk_t prev = 0;
377 int len = 0;
Aneesh Kumar K.V56b19862009-03-12 09:51:20 -0400378 while (entries) {
379 if (!ext4_valid_extent(inode, ext))
380 return 0;
Eryu Guan5946d082013-12-03 21:22:21 -0500381
382 /* Check for overlapping extents */
383 lblock = le32_to_cpu(ext->ee_block);
384 len = ext4_ext_get_actual_len(ext);
385 if ((lblock <= prev) && prev) {
386 pblock = ext4_ext_pblock(ext);
387 es->s_last_error_block = cpu_to_le64(pblock);
388 return 0;
389 }
Aneesh Kumar K.V56b19862009-03-12 09:51:20 -0400390 ext++;
391 entries--;
Eryu Guan5946d082013-12-03 21:22:21 -0500392 prev = lblock + len - 1;
Aneesh Kumar K.V56b19862009-03-12 09:51:20 -0400393 }
394 } else {
Yongqiang Yang81fdbb42011-10-29 09:23:38 -0400395 struct ext4_extent_idx *ext_idx = EXT_FIRST_INDEX(eh);
Aneesh Kumar K.V56b19862009-03-12 09:51:20 -0400396 while (entries) {
397 if (!ext4_valid_extent_idx(inode, ext_idx))
398 return 0;
399 ext_idx++;
400 entries--;
401 }
402 }
403 return 1;
404}
405
Theodore Ts'oc398eda2010-07-27 11:56:40 -0400406static int __ext4_ext_check(const char *function, unsigned int line,
407 struct inode *inode, struct ext4_extent_header *eh,
Theodore Ts'oc3491792013-08-16 21:21:41 -0400408 int depth, ext4_fsblk_t pblk)
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400409{
410 const char *error_msg;
Darrick J. Wong6a797d22015-10-17 16:16:04 -0400411 int max = 0, err = -EFSCORRUPTED;
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400412
413 if (unlikely(eh->eh_magic != EXT4_EXT_MAGIC)) {
414 error_msg = "invalid magic";
415 goto corrupted;
416 }
417 if (unlikely(le16_to_cpu(eh->eh_depth) != depth)) {
418 error_msg = "unexpected eh_depth";
419 goto corrupted;
420 }
421 if (unlikely(eh->eh_max == 0)) {
422 error_msg = "invalid eh_max";
423 goto corrupted;
424 }
425 max = ext4_ext_max_entries(inode, depth);
426 if (unlikely(le16_to_cpu(eh->eh_max) > max)) {
427 error_msg = "too large eh_max";
428 goto corrupted;
429 }
430 if (unlikely(le16_to_cpu(eh->eh_entries) > le16_to_cpu(eh->eh_max))) {
431 error_msg = "invalid eh_entries";
432 goto corrupted;
433 }
Aneesh Kumar K.V56b19862009-03-12 09:51:20 -0400434 if (!ext4_valid_extent_entries(inode, eh, depth)) {
435 error_msg = "invalid extent entries";
436 goto corrupted;
437 }
Vegard Nossum7bc94912016-07-15 00:22:07 -0400438 if (unlikely(depth > 32)) {
439 error_msg = "too large eh_depth";
440 goto corrupted;
441 }
Darrick J. Wong7ac59902012-04-29 18:37:10 -0400442 /* Verify checksum on non-root extent tree nodes */
443 if (ext_depth(inode) != depth &&
444 !ext4_extent_block_csum_verify(inode, eh)) {
445 error_msg = "extent tree corrupted";
Darrick J. Wong6a797d22015-10-17 16:16:04 -0400446 err = -EFSBADCRC;
Darrick J. Wong7ac59902012-04-29 18:37:10 -0400447 goto corrupted;
448 }
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400449 return 0;
450
451corrupted:
Theodore Ts'o878520a2019-11-19 21:54:15 -0500452 ext4_set_errno(inode->i_sb, -err);
Theodore Ts'oc398eda2010-07-27 11:56:40 -0400453 ext4_error_inode(inode, function, line, 0,
Theodore Ts'oc3491792013-08-16 21:21:41 -0400454 "pblk %llu bad header/extent: %s - magic %x, "
455 "entries %u, max %u(%u), depth %u(%u)",
456 (unsigned long long) pblk, error_msg,
457 le16_to_cpu(eh->eh_magic),
458 le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max),
459 max, le16_to_cpu(eh->eh_depth), depth);
Darrick J. Wong6a797d22015-10-17 16:16:04 -0400460 return err;
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400461}
462
Theodore Ts'oc3491792013-08-16 21:21:41 -0400463#define ext4_ext_check(inode, eh, depth, pblk) \
464 __ext4_ext_check(__func__, __LINE__, (inode), (eh), (depth), (pblk))
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400465
Aneesh Kumar K.V7a262f72009-03-27 16:39:58 -0400466int ext4_ext_check_inode(struct inode *inode)
467{
Theodore Ts'oc3491792013-08-16 21:21:41 -0400468 return ext4_ext_check(inode, ext_inode_hdr(inode), ext_depth(inode), 0);
Aneesh Kumar K.V7a262f72009-03-27 16:39:58 -0400469}
470
Theodore Ts'o7d7ea892013-08-16 21:20:41 -0400471static struct buffer_head *
472__read_extent_tree_block(const char *function, unsigned int line,
Theodore Ts'o107a7bd2013-08-16 21:23:41 -0400473 struct inode *inode, ext4_fsblk_t pblk, int depth,
474 int flags)
Darrick J. Wongf8489122012-04-29 18:21:10 -0400475{
Theodore Ts'o7d7ea892013-08-16 21:20:41 -0400476 struct buffer_head *bh;
477 int err;
Darrick J. Wongf8489122012-04-29 18:21:10 -0400478
Nikolay Borisovc45653c2015-07-02 01:34:07 -0400479 bh = sb_getblk_gfp(inode->i_sb, pblk, __GFP_MOVABLE | GFP_NOFS);
Theodore Ts'o7d7ea892013-08-16 21:20:41 -0400480 if (unlikely(!bh))
481 return ERR_PTR(-ENOMEM);
482
483 if (!bh_uptodate_or_lock(bh)) {
484 trace_ext4_ext_load_extent(inode, pblk, _RET_IP_);
485 err = bh_submit_read(bh);
486 if (err < 0)
487 goto errout;
488 }
Theodore Ts'o7869a4a2013-08-16 22:05:14 -0400489 if (buffer_verified(bh) && !(flags & EXT4_EX_FORCE_CACHE))
Theodore Ts'o7d7ea892013-08-16 21:20:41 -0400490 return bh;
Theodore Ts'o0a944e82019-05-22 10:27:01 -0400491 if (!ext4_has_feature_journal(inode->i_sb) ||
492 (inode->i_ino !=
493 le32_to_cpu(EXT4_SB(inode->i_sb)->s_es->s_journal_inum))) {
494 err = __ext4_ext_check(function, line, inode,
495 ext_block_hdr(bh), depth, pblk);
496 if (err)
497 goto errout;
498 }
Darrick J. Wongf8489122012-04-29 18:21:10 -0400499 set_buffer_verified(bh);
Theodore Ts'o107a7bd2013-08-16 21:23:41 -0400500 /*
501 * If this is a leaf block, cache all of its entries
502 */
503 if (!(flags & EXT4_EX_NOCACHE) && depth == 0) {
504 struct ext4_extent_header *eh = ext_block_hdr(bh);
505 struct ext4_extent *ex = EXT_FIRST_EXTENT(eh);
506 ext4_lblk_t prev = 0;
507 int i;
508
509 for (i = le16_to_cpu(eh->eh_entries); i > 0; i--, ex++) {
510 unsigned int status = EXTENT_STATUS_WRITTEN;
511 ext4_lblk_t lblk = le32_to_cpu(ex->ee_block);
512 int len = ext4_ext_get_actual_len(ex);
513
514 if (prev && (prev != lblk))
515 ext4_es_cache_extent(inode, prev,
516 lblk - prev, ~0,
517 EXTENT_STATUS_HOLE);
518
Lukas Czerner556615d2014-04-20 23:45:47 -0400519 if (ext4_ext_is_unwritten(ex))
Theodore Ts'o107a7bd2013-08-16 21:23:41 -0400520 status = EXTENT_STATUS_UNWRITTEN;
521 ext4_es_cache_extent(inode, lblk, len,
522 ext4_ext_pblock(ex), status);
523 prev = lblk + len;
524 }
525 }
Theodore Ts'o7d7ea892013-08-16 21:20:41 -0400526 return bh;
527errout:
528 put_bh(bh);
529 return ERR_PTR(err);
530
Darrick J. Wongf8489122012-04-29 18:21:10 -0400531}
532
Theodore Ts'o107a7bd2013-08-16 21:23:41 -0400533#define read_extent_tree_block(inode, pblk, depth, flags) \
534 __read_extent_tree_block(__func__, __LINE__, (inode), (pblk), \
535 (depth), (flags))
Darrick J. Wongf8489122012-04-29 18:21:10 -0400536
Theodore Ts'o7869a4a2013-08-16 22:05:14 -0400537/*
538 * This function is called to cache a file's extent information in the
539 * extent status tree
540 */
541int ext4_ext_precache(struct inode *inode)
542{
543 struct ext4_inode_info *ei = EXT4_I(inode);
544 struct ext4_ext_path *path = NULL;
545 struct buffer_head *bh;
546 int i = 0, depth, ret = 0;
547
548 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
549 return 0; /* not an extent-mapped inode */
550
551 down_read(&ei->i_data_sem);
552 depth = ext_depth(inode);
553
Kees Cook6396bb22018-06-12 14:03:40 -0700554 path = kcalloc(depth + 1, sizeof(struct ext4_ext_path),
Theodore Ts'o7869a4a2013-08-16 22:05:14 -0400555 GFP_NOFS);
556 if (path == NULL) {
557 up_read(&ei->i_data_sem);
558 return -ENOMEM;
559 }
560
561 /* Don't cache anything if there are no external extent blocks */
562 if (depth == 0)
563 goto out;
564 path[0].p_hdr = ext_inode_hdr(inode);
565 ret = ext4_ext_check(inode, path[0].p_hdr, depth, 0);
566 if (ret)
567 goto out;
568 path[0].p_idx = EXT_FIRST_INDEX(path[0].p_hdr);
569 while (i >= 0) {
570 /*
571 * If this is a leaf block or we've reached the end of
572 * the index block, go up
573 */
574 if ((i == depth) ||
575 path[i].p_idx > EXT_LAST_INDEX(path[i].p_hdr)) {
576 brelse(path[i].p_bh);
577 path[i].p_bh = NULL;
578 i--;
579 continue;
580 }
581 bh = read_extent_tree_block(inode,
582 ext4_idx_pblock(path[i].p_idx++),
583 depth - i - 1,
584 EXT4_EX_FORCE_CACHE);
585 if (IS_ERR(bh)) {
586 ret = PTR_ERR(bh);
587 break;
588 }
589 i++;
590 path[i].p_bh = bh;
591 path[i].p_hdr = ext_block_hdr(bh);
592 path[i].p_idx = EXT_FIRST_INDEX(path[i].p_hdr);
593 }
594 ext4_set_inode_state(inode, EXT4_STATE_EXT_PRECACHED);
595out:
596 up_read(&ei->i_data_sem);
597 ext4_ext_drop_refs(path);
598 kfree(path);
599 return ret;
600}
601
Alex Tomasa86c6182006-10-11 01:21:03 -0700602#ifdef EXT_DEBUG
603static void ext4_ext_show_path(struct inode *inode, struct ext4_ext_path *path)
604{
605 int k, l = path->p_depth;
606
607 ext_debug("path:");
608 for (k = 0; k <= l; k++, path++) {
609 if (path->p_idx) {
Mingming Cao2ae02102006-10-11 01:21:11 -0700610 ext_debug(" %d->%llu", le32_to_cpu(path->p_idx->ei_block),
Theodore Ts'obf89d162010-10-27 21:30:14 -0400611 ext4_idx_pblock(path->p_idx));
Alex Tomasa86c6182006-10-11 01:21:03 -0700612 } else if (path->p_ext) {
Mingming553f9002009-09-18 13:34:55 -0400613 ext_debug(" %d:[%d]%d:%llu ",
Alex Tomasa86c6182006-10-11 01:21:03 -0700614 le32_to_cpu(path->p_ext->ee_block),
Lukas Czerner556615d2014-04-20 23:45:47 -0400615 ext4_ext_is_unwritten(path->p_ext),
Amit Aroraa2df2a62007-07-17 21:42:41 -0400616 ext4_ext_get_actual_len(path->p_ext),
Theodore Ts'obf89d162010-10-27 21:30:14 -0400617 ext4_ext_pblock(path->p_ext));
Alex Tomasa86c6182006-10-11 01:21:03 -0700618 } else
619 ext_debug(" []");
620 }
621 ext_debug("\n");
622}
623
624static void ext4_ext_show_leaf(struct inode *inode, struct ext4_ext_path *path)
625{
626 int depth = ext_depth(inode);
627 struct ext4_extent_header *eh;
628 struct ext4_extent *ex;
629 int i;
630
631 if (!path)
632 return;
633
634 eh = path[depth].p_hdr;
635 ex = EXT_FIRST_EXTENT(eh);
636
Mingming553f9002009-09-18 13:34:55 -0400637 ext_debug("Displaying leaf extents for inode %lu\n", inode->i_ino);
638
Alex Tomasa86c6182006-10-11 01:21:03 -0700639 for (i = 0; i < le16_to_cpu(eh->eh_entries); i++, ex++) {
Mingming553f9002009-09-18 13:34:55 -0400640 ext_debug("%d:[%d]%d:%llu ", le32_to_cpu(ex->ee_block),
Lukas Czerner556615d2014-04-20 23:45:47 -0400641 ext4_ext_is_unwritten(ex),
Theodore Ts'obf89d162010-10-27 21:30:14 -0400642 ext4_ext_get_actual_len(ex), ext4_ext_pblock(ex));
Alex Tomasa86c6182006-10-11 01:21:03 -0700643 }
644 ext_debug("\n");
645}
Yongqiang Yang1b16da72011-05-25 17:41:48 -0400646
647static void ext4_ext_show_move(struct inode *inode, struct ext4_ext_path *path,
648 ext4_fsblk_t newblock, int level)
649{
650 int depth = ext_depth(inode);
651 struct ext4_extent *ex;
652
653 if (depth != level) {
654 struct ext4_extent_idx *idx;
655 idx = path[level].p_idx;
656 while (idx <= EXT_MAX_INDEX(path[level].p_hdr)) {
657 ext_debug("%d: move %d:%llu in new index %llu\n", level,
658 le32_to_cpu(idx->ei_block),
659 ext4_idx_pblock(idx),
660 newblock);
661 idx++;
662 }
663
664 return;
665 }
666
667 ex = path[depth].p_ext;
668 while (ex <= EXT_MAX_EXTENT(path[depth].p_hdr)) {
669 ext_debug("move %d:%llu:[%d]%d in new leaf %llu\n",
670 le32_to_cpu(ex->ee_block),
671 ext4_ext_pblock(ex),
Lukas Czerner556615d2014-04-20 23:45:47 -0400672 ext4_ext_is_unwritten(ex),
Yongqiang Yang1b16da72011-05-25 17:41:48 -0400673 ext4_ext_get_actual_len(ex),
674 newblock);
675 ex++;
676 }
677}
678
Alex Tomasa86c6182006-10-11 01:21:03 -0700679#else
Theodore Ts'oaf5bc922008-09-08 22:25:24 -0400680#define ext4_ext_show_path(inode, path)
681#define ext4_ext_show_leaf(inode, path)
Yongqiang Yang1b16da72011-05-25 17:41:48 -0400682#define ext4_ext_show_move(inode, path, newblock, level)
Alex Tomasa86c6182006-10-11 01:21:03 -0700683#endif
684
Aneesh Kumar K.Vb35905c2008-02-25 16:54:37 -0500685void ext4_ext_drop_refs(struct ext4_ext_path *path)
Alex Tomasa86c6182006-10-11 01:21:03 -0700686{
Theodore Ts'ob7ea89a2014-09-01 14:39:09 -0400687 int depth, i;
Alex Tomasa86c6182006-10-11 01:21:03 -0700688
Theodore Ts'ob7ea89a2014-09-01 14:39:09 -0400689 if (!path)
690 return;
691 depth = path->p_depth;
Alex Tomasa86c6182006-10-11 01:21:03 -0700692 for (i = 0; i <= depth; i++, path++)
693 if (path->p_bh) {
694 brelse(path->p_bh);
695 path->p_bh = NULL;
696 }
697}
698
699/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -0700700 * ext4_ext_binsearch_idx:
701 * binary search for the closest index of the given block
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400702 * the header must be checked before calling this
Alex Tomasa86c6182006-10-11 01:21:03 -0700703 */
704static void
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -0500705ext4_ext_binsearch_idx(struct inode *inode,
706 struct ext4_ext_path *path, ext4_lblk_t block)
Alex Tomasa86c6182006-10-11 01:21:03 -0700707{
708 struct ext4_extent_header *eh = path->p_hdr;
709 struct ext4_extent_idx *r, *l, *m;
710
Alex Tomasa86c6182006-10-11 01:21:03 -0700711
Eric Sandeenbba90742008-01-28 23:58:27 -0500712 ext_debug("binsearch for %u(idx): ", block);
Alex Tomasa86c6182006-10-11 01:21:03 -0700713
714 l = EXT_FIRST_INDEX(eh) + 1;
Dmitry Monakhove9f410b2007-07-18 09:09:15 -0400715 r = EXT_LAST_INDEX(eh);
Alex Tomasa86c6182006-10-11 01:21:03 -0700716 while (l <= r) {
717 m = l + (r - l) / 2;
718 if (block < le32_to_cpu(m->ei_block))
719 r = m - 1;
720 else
721 l = m + 1;
Dmitry Monakhov26d535e2007-07-18 08:33:37 -0400722 ext_debug("%p(%u):%p(%u):%p(%u) ", l, le32_to_cpu(l->ei_block),
723 m, le32_to_cpu(m->ei_block),
724 r, le32_to_cpu(r->ei_block));
Alex Tomasa86c6182006-10-11 01:21:03 -0700725 }
726
727 path->p_idx = l - 1;
Zheng Liu4a3c3a52012-05-28 17:55:16 -0400728 ext_debug(" -> %u->%lld ", le32_to_cpu(path->p_idx->ei_block),
Theodore Ts'obf89d162010-10-27 21:30:14 -0400729 ext4_idx_pblock(path->p_idx));
Alex Tomasa86c6182006-10-11 01:21:03 -0700730
731#ifdef CHECK_BINSEARCH
732 {
733 struct ext4_extent_idx *chix, *ix;
734 int k;
735
736 chix = ix = EXT_FIRST_INDEX(eh);
737 for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ix++) {
738 if (k != 0 &&
739 le32_to_cpu(ix->ei_block) <= le32_to_cpu(ix[-1].ei_block)) {
Theodore Ts'o4776004f2008-09-08 23:00:52 -0400740 printk(KERN_DEBUG "k=%d, ix=0x%p, "
741 "first=0x%p\n", k,
742 ix, EXT_FIRST_INDEX(eh));
743 printk(KERN_DEBUG "%u <= %u\n",
Alex Tomasa86c6182006-10-11 01:21:03 -0700744 le32_to_cpu(ix->ei_block),
745 le32_to_cpu(ix[-1].ei_block));
746 }
747 BUG_ON(k && le32_to_cpu(ix->ei_block)
Dave Kleikamp8c55e202007-05-24 13:04:54 -0400748 <= le32_to_cpu(ix[-1].ei_block));
Alex Tomasa86c6182006-10-11 01:21:03 -0700749 if (block < le32_to_cpu(ix->ei_block))
750 break;
751 chix = ix;
752 }
753 BUG_ON(chix != path->p_idx);
754 }
755#endif
756
757}
758
759/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -0700760 * ext4_ext_binsearch:
761 * binary search for closest extent of the given block
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400762 * the header must be checked before calling this
Alex Tomasa86c6182006-10-11 01:21:03 -0700763 */
764static void
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -0500765ext4_ext_binsearch(struct inode *inode,
766 struct ext4_ext_path *path, ext4_lblk_t block)
Alex Tomasa86c6182006-10-11 01:21:03 -0700767{
768 struct ext4_extent_header *eh = path->p_hdr;
769 struct ext4_extent *r, *l, *m;
770
Alex Tomasa86c6182006-10-11 01:21:03 -0700771 if (eh->eh_entries == 0) {
772 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -0700773 * this leaf is empty:
774 * we get such a leaf in split/add case
Alex Tomasa86c6182006-10-11 01:21:03 -0700775 */
776 return;
777 }
778
Eric Sandeenbba90742008-01-28 23:58:27 -0500779 ext_debug("binsearch for %u: ", block);
Alex Tomasa86c6182006-10-11 01:21:03 -0700780
781 l = EXT_FIRST_EXTENT(eh) + 1;
Dmitry Monakhove9f410b2007-07-18 09:09:15 -0400782 r = EXT_LAST_EXTENT(eh);
Alex Tomasa86c6182006-10-11 01:21:03 -0700783
784 while (l <= r) {
785 m = l + (r - l) / 2;
786 if (block < le32_to_cpu(m->ee_block))
787 r = m - 1;
788 else
789 l = m + 1;
Dmitry Monakhov26d535e2007-07-18 08:33:37 -0400790 ext_debug("%p(%u):%p(%u):%p(%u) ", l, le32_to_cpu(l->ee_block),
791 m, le32_to_cpu(m->ee_block),
792 r, le32_to_cpu(r->ee_block));
Alex Tomasa86c6182006-10-11 01:21:03 -0700793 }
794
795 path->p_ext = l - 1;
Mingming553f9002009-09-18 13:34:55 -0400796 ext_debug(" -> %d:%llu:[%d]%d ",
Dave Kleikamp8c55e202007-05-24 13:04:54 -0400797 le32_to_cpu(path->p_ext->ee_block),
Theodore Ts'obf89d162010-10-27 21:30:14 -0400798 ext4_ext_pblock(path->p_ext),
Lukas Czerner556615d2014-04-20 23:45:47 -0400799 ext4_ext_is_unwritten(path->p_ext),
Amit Aroraa2df2a62007-07-17 21:42:41 -0400800 ext4_ext_get_actual_len(path->p_ext));
Alex Tomasa86c6182006-10-11 01:21:03 -0700801
802#ifdef CHECK_BINSEARCH
803 {
804 struct ext4_extent *chex, *ex;
805 int k;
806
807 chex = ex = EXT_FIRST_EXTENT(eh);
808 for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ex++) {
809 BUG_ON(k && le32_to_cpu(ex->ee_block)
Dave Kleikamp8c55e202007-05-24 13:04:54 -0400810 <= le32_to_cpu(ex[-1].ee_block));
Alex Tomasa86c6182006-10-11 01:21:03 -0700811 if (block < le32_to_cpu(ex->ee_block))
812 break;
813 chex = ex;
814 }
815 BUG_ON(chex != path->p_ext);
816 }
817#endif
818
819}
820
821int ext4_ext_tree_init(handle_t *handle, struct inode *inode)
822{
823 struct ext4_extent_header *eh;
824
825 eh = ext_inode_hdr(inode);
826 eh->eh_depth = 0;
827 eh->eh_entries = 0;
828 eh->eh_magic = EXT4_EXT_MAGIC;
Theodore Ts'o55ad63b2009-08-28 10:40:33 -0400829 eh->eh_max = cpu_to_le16(ext4_ext_space_root(inode, 0));
Alex Tomasa86c6182006-10-11 01:21:03 -0700830 ext4_mark_inode_dirty(handle, inode);
Alex Tomasa86c6182006-10-11 01:21:03 -0700831 return 0;
832}
833
834struct ext4_ext_path *
Theodore Ts'oed8a1a72014-09-01 14:43:09 -0400835ext4_find_extent(struct inode *inode, ext4_lblk_t block,
836 struct ext4_ext_path **orig_path, int flags)
Alex Tomasa86c6182006-10-11 01:21:03 -0700837{
838 struct ext4_extent_header *eh;
839 struct buffer_head *bh;
Theodore Ts'o705912c2014-09-01 14:34:09 -0400840 struct ext4_ext_path *path = orig_path ? *orig_path : NULL;
841 short int depth, i, ppos = 0;
Theodore Ts'o860d21e2013-01-12 16:19:36 -0500842 int ret;
Alex Tomasa86c6182006-10-11 01:21:03 -0700843
844 eh = ext_inode_hdr(inode);
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400845 depth = ext_depth(inode);
Theodore Ts'obc890a62018-06-14 12:55:10 -0400846 if (depth < 0 || depth > EXT4_MAX_EXTENT_DEPTH) {
847 EXT4_ERROR_INODE(inode, "inode has invalid extent depth: %d",
848 depth);
849 ret = -EFSCORRUPTED;
850 goto err;
851 }
Alex Tomasa86c6182006-10-11 01:21:03 -0700852
Theodore Ts'o10809df82014-09-01 14:40:09 -0400853 if (path) {
Theodore Ts'o523f4312014-09-01 14:38:09 -0400854 ext4_ext_drop_refs(path);
Theodore Ts'o10809df82014-09-01 14:40:09 -0400855 if (depth > path[0].p_maxdepth) {
856 kfree(path);
857 *orig_path = path = NULL;
858 }
859 }
860 if (!path) {
Theodore Ts'o523f4312014-09-01 14:38:09 -0400861 /* account possible depth increase */
Kees Cook6396bb22018-06-12 14:03:40 -0700862 path = kcalloc(depth + 2, sizeof(struct ext4_ext_path),
Alex Tomasa86c6182006-10-11 01:21:03 -0700863 GFP_NOFS);
Theodore Ts'o19008f62014-08-31 15:03:14 -0400864 if (unlikely(!path))
Alex Tomasa86c6182006-10-11 01:21:03 -0700865 return ERR_PTR(-ENOMEM);
Theodore Ts'o10809df82014-09-01 14:40:09 -0400866 path[0].p_maxdepth = depth + 1;
Alex Tomasa86c6182006-10-11 01:21:03 -0700867 }
Alex Tomasa86c6182006-10-11 01:21:03 -0700868 path[0].p_hdr = eh;
Shen Feng1973adc2008-07-11 19:27:31 -0400869 path[0].p_bh = NULL;
Alex Tomasa86c6182006-10-11 01:21:03 -0700870
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400871 i = depth;
Alex Tomasa86c6182006-10-11 01:21:03 -0700872 /* walk through the tree */
873 while (i) {
874 ext_debug("depth %d: num %d, max %d\n",
875 ppos, le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max));
Alex Tomasc29c0ae2007-07-18 09:19:09 -0400876
Alex Tomasa86c6182006-10-11 01:21:03 -0700877 ext4_ext_binsearch_idx(inode, path + ppos, block);
Theodore Ts'obf89d162010-10-27 21:30:14 -0400878 path[ppos].p_block = ext4_idx_pblock(path[ppos].p_idx);
Alex Tomasa86c6182006-10-11 01:21:03 -0700879 path[ppos].p_depth = i;
880 path[ppos].p_ext = NULL;
881
Theodore Ts'o107a7bd2013-08-16 21:23:41 -0400882 bh = read_extent_tree_block(inode, path[ppos].p_block, --i,
883 flags);
Viresh Kumara1c83682015-08-12 15:59:44 +0530884 if (IS_ERR(bh)) {
Theodore Ts'o7d7ea892013-08-16 21:20:41 -0400885 ret = PTR_ERR(bh);
Alex Tomasa86c6182006-10-11 01:21:03 -0700886 goto err;
Theodore Ts'o860d21e2013-01-12 16:19:36 -0500887 }
Theodore Ts'o7d7ea892013-08-16 21:20:41 -0400888
Alex Tomasa86c6182006-10-11 01:21:03 -0700889 eh = ext_block_hdr(bh);
890 ppos++;
Alex Tomasa86c6182006-10-11 01:21:03 -0700891 path[ppos].p_bh = bh;
892 path[ppos].p_hdr = eh;
Alex Tomasa86c6182006-10-11 01:21:03 -0700893 }
894
895 path[ppos].p_depth = i;
Alex Tomasa86c6182006-10-11 01:21:03 -0700896 path[ppos].p_ext = NULL;
897 path[ppos].p_idx = NULL;
898
Alex Tomasa86c6182006-10-11 01:21:03 -0700899 /* find extent */
900 ext4_ext_binsearch(inode, path + ppos, block);
Shen Feng1973adc2008-07-11 19:27:31 -0400901 /* if not an empty leaf */
902 if (path[ppos].p_ext)
Theodore Ts'obf89d162010-10-27 21:30:14 -0400903 path[ppos].p_block = ext4_ext_pblock(path[ppos].p_ext);
Alex Tomasa86c6182006-10-11 01:21:03 -0700904
905 ext4_ext_show_path(inode, path);
906
907 return path;
908
909err:
910 ext4_ext_drop_refs(path);
Theodore Ts'odfe50802014-09-01 14:37:09 -0400911 kfree(path);
912 if (orig_path)
913 *orig_path = NULL;
Theodore Ts'o860d21e2013-01-12 16:19:36 -0500914 return ERR_PTR(ret);
Alex Tomasa86c6182006-10-11 01:21:03 -0700915}
916
917/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -0700918 * ext4_ext_insert_index:
919 * insert new index [@logical;@ptr] into the block at @curp;
920 * check where to insert: before @curp or after @curp
Alex Tomasa86c6182006-10-11 01:21:03 -0700921 */
Theodore Ts'o1f109d52010-10-27 21:30:14 -0400922static int ext4_ext_insert_index(handle_t *handle, struct inode *inode,
923 struct ext4_ext_path *curp,
924 int logical, ext4_fsblk_t ptr)
Alex Tomasa86c6182006-10-11 01:21:03 -0700925{
926 struct ext4_extent_idx *ix;
927 int len, err;
928
Avantika Mathur7e028972006-12-06 20:41:33 -0800929 err = ext4_ext_get_access(handle, inode, curp);
930 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -0700931 return err;
932
Frank Mayhar273df552010-03-02 11:46:09 -0500933 if (unlikely(logical == le32_to_cpu(curp->p_idx->ei_block))) {
934 EXT4_ERROR_INODE(inode,
935 "logical %d == ei_block %d!",
936 logical, le32_to_cpu(curp->p_idx->ei_block));
Darrick J. Wong6a797d22015-10-17 16:16:04 -0400937 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -0500938 }
Robin Dongd4620312011-07-17 23:43:42 -0400939
940 if (unlikely(le16_to_cpu(curp->p_hdr->eh_entries)
941 >= le16_to_cpu(curp->p_hdr->eh_max))) {
942 EXT4_ERROR_INODE(inode,
943 "eh_entries %d >= eh_max %d!",
944 le16_to_cpu(curp->p_hdr->eh_entries),
945 le16_to_cpu(curp->p_hdr->eh_max));
Darrick J. Wong6a797d22015-10-17 16:16:04 -0400946 return -EFSCORRUPTED;
Robin Dongd4620312011-07-17 23:43:42 -0400947 }
948
Alex Tomasa86c6182006-10-11 01:21:03 -0700949 if (logical > le32_to_cpu(curp->p_idx->ei_block)) {
950 /* insert after */
Eric Gouriou80e675f2011-10-27 11:52:18 -0400951 ext_debug("insert new index %d after: %llu\n", logical, ptr);
Alex Tomasa86c6182006-10-11 01:21:03 -0700952 ix = curp->p_idx + 1;
953 } else {
954 /* insert before */
Eric Gouriou80e675f2011-10-27 11:52:18 -0400955 ext_debug("insert new index %d before: %llu\n", logical, ptr);
Alex Tomasa86c6182006-10-11 01:21:03 -0700956 ix = curp->p_idx;
957 }
958
Eric Gouriou80e675f2011-10-27 11:52:18 -0400959 len = EXT_LAST_INDEX(curp->p_hdr) - ix + 1;
960 BUG_ON(len < 0);
961 if (len > 0) {
962 ext_debug("insert new index %d: "
963 "move %d indices from 0x%p to 0x%p\n",
964 logical, len, ix, ix + 1);
965 memmove(ix + 1, ix, len * sizeof(struct ext4_extent_idx));
966 }
967
Tao Maf472e022011-10-17 10:13:46 -0400968 if (unlikely(ix > EXT_MAX_INDEX(curp->p_hdr))) {
969 EXT4_ERROR_INODE(inode, "ix > EXT_MAX_INDEX!");
Darrick J. Wong6a797d22015-10-17 16:16:04 -0400970 return -EFSCORRUPTED;
Tao Maf472e022011-10-17 10:13:46 -0400971 }
972
Alex Tomasa86c6182006-10-11 01:21:03 -0700973 ix->ei_block = cpu_to_le32(logical);
Alex Tomasf65e6fb2006-10-11 01:21:05 -0700974 ext4_idx_store_pblock(ix, ptr);
Marcin Slusarze8546d02008-04-17 10:38:59 -0400975 le16_add_cpu(&curp->p_hdr->eh_entries, 1);
Alex Tomasa86c6182006-10-11 01:21:03 -0700976
Frank Mayhar273df552010-03-02 11:46:09 -0500977 if (unlikely(ix > EXT_LAST_INDEX(curp->p_hdr))) {
978 EXT4_ERROR_INODE(inode, "ix > EXT_LAST_INDEX!");
Darrick J. Wong6a797d22015-10-17 16:16:04 -0400979 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -0500980 }
Alex Tomasa86c6182006-10-11 01:21:03 -0700981
982 err = ext4_ext_dirty(handle, inode, curp);
983 ext4_std_error(inode->i_sb, err);
984
985 return err;
986}
987
988/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -0700989 * ext4_ext_split:
990 * inserts new subtree into the path, using free index entry
991 * at depth @at:
992 * - allocates all needed blocks (new leaf and all intermediate index blocks)
993 * - makes decision where to split
994 * - moves remaining extents and index entries (right to the split point)
995 * into the newly allocated blocks
996 * - initializes subtree
Alex Tomasa86c6182006-10-11 01:21:03 -0700997 */
998static int ext4_ext_split(handle_t *handle, struct inode *inode,
Allison Henderson55f020d2011-05-25 07:41:26 -0400999 unsigned int flags,
1000 struct ext4_ext_path *path,
1001 struct ext4_extent *newext, int at)
Alex Tomasa86c6182006-10-11 01:21:03 -07001002{
1003 struct buffer_head *bh = NULL;
1004 int depth = ext_depth(inode);
1005 struct ext4_extent_header *neh;
1006 struct ext4_extent_idx *fidx;
Alex Tomasa86c6182006-10-11 01:21:03 -07001007 int i = at, k, m, a;
Alex Tomasf65e6fb2006-10-11 01:21:05 -07001008 ext4_fsblk_t newblock, oldblock;
Alex Tomasa86c6182006-10-11 01:21:03 -07001009 __le32 border;
Alex Tomasf65e6fb2006-10-11 01:21:05 -07001010 ext4_fsblk_t *ablocks = NULL; /* array of allocated blocks */
Alex Tomasa86c6182006-10-11 01:21:03 -07001011 int err = 0;
Sriram Rajagopalan592acbf2019-05-10 19:28:06 -04001012 size_t ext_size = 0;
Alex Tomasa86c6182006-10-11 01:21:03 -07001013
1014 /* make decision: where to split? */
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001015 /* FIXME: now decision is simplest: at current extent */
Alex Tomasa86c6182006-10-11 01:21:03 -07001016
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001017 /* if current leaf will be split, then we should use
Alex Tomasa86c6182006-10-11 01:21:03 -07001018 * border from split point */
Frank Mayhar273df552010-03-02 11:46:09 -05001019 if (unlikely(path[depth].p_ext > EXT_MAX_EXTENT(path[depth].p_hdr))) {
1020 EXT4_ERROR_INODE(inode, "p_ext > EXT_MAX_EXTENT!");
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001021 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001022 }
Alex Tomasa86c6182006-10-11 01:21:03 -07001023 if (path[depth].p_ext != EXT_MAX_EXTENT(path[depth].p_hdr)) {
1024 border = path[depth].p_ext[1].ee_block;
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001025 ext_debug("leaf will be split."
Alex Tomasa86c6182006-10-11 01:21:03 -07001026 " next leaf starts at %d\n",
Dave Kleikamp8c55e202007-05-24 13:04:54 -04001027 le32_to_cpu(border));
Alex Tomasa86c6182006-10-11 01:21:03 -07001028 } else {
1029 border = newext->ee_block;
1030 ext_debug("leaf will be added."
1031 " next leaf starts at %d\n",
Dave Kleikamp8c55e202007-05-24 13:04:54 -04001032 le32_to_cpu(border));
Alex Tomasa86c6182006-10-11 01:21:03 -07001033 }
1034
1035 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001036 * If error occurs, then we break processing
1037 * and mark filesystem read-only. index won't
Alex Tomasa86c6182006-10-11 01:21:03 -07001038 * be inserted and tree will be in consistent
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001039 * state. Next mount will repair buffers too.
Alex Tomasa86c6182006-10-11 01:21:03 -07001040 */
1041
1042 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001043 * Get array to track all allocated blocks.
1044 * We need this to handle errors and free blocks
1045 * upon them.
Alex Tomasa86c6182006-10-11 01:21:03 -07001046 */
Kees Cook6396bb22018-06-12 14:03:40 -07001047 ablocks = kcalloc(depth, sizeof(ext4_fsblk_t), GFP_NOFS);
Alex Tomasa86c6182006-10-11 01:21:03 -07001048 if (!ablocks)
1049 return -ENOMEM;
Alex Tomasa86c6182006-10-11 01:21:03 -07001050
1051 /* allocate all needed blocks */
1052 ext_debug("allocate %d blocks for indexes/leaf\n", depth - at);
1053 for (a = 0; a < depth - at; a++) {
Aneesh Kumar K.V654b4902008-07-11 19:27:31 -04001054 newblock = ext4_ext_new_meta_block(handle, inode, path,
Allison Henderson55f020d2011-05-25 07:41:26 -04001055 newext, &err, flags);
Alex Tomasa86c6182006-10-11 01:21:03 -07001056 if (newblock == 0)
1057 goto cleanup;
1058 ablocks[a] = newblock;
1059 }
1060
1061 /* initialize new leaf */
1062 newblock = ablocks[--a];
Frank Mayhar273df552010-03-02 11:46:09 -05001063 if (unlikely(newblock == 0)) {
1064 EXT4_ERROR_INODE(inode, "newblock == 0!");
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001065 err = -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001066 goto cleanup;
1067 }
Nikolay Borisovc45653c2015-07-02 01:34:07 -04001068 bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS);
Wang Shilongaebf0242013-01-12 16:28:47 -05001069 if (unlikely(!bh)) {
Theodore Ts'o860d21e2013-01-12 16:19:36 -05001070 err = -ENOMEM;
Alex Tomasa86c6182006-10-11 01:21:03 -07001071 goto cleanup;
1072 }
1073 lock_buffer(bh);
1074
Avantika Mathur7e028972006-12-06 20:41:33 -08001075 err = ext4_journal_get_create_access(handle, bh);
1076 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07001077 goto cleanup;
1078
1079 neh = ext_block_hdr(bh);
1080 neh->eh_entries = 0;
Theodore Ts'o55ad63b2009-08-28 10:40:33 -04001081 neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0));
Alex Tomasa86c6182006-10-11 01:21:03 -07001082 neh->eh_magic = EXT4_EXT_MAGIC;
1083 neh->eh_depth = 0;
Alex Tomasa86c6182006-10-11 01:21:03 -07001084
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001085 /* move remainder of path[depth] to the new leaf */
Frank Mayhar273df552010-03-02 11:46:09 -05001086 if (unlikely(path[depth].p_hdr->eh_entries !=
1087 path[depth].p_hdr->eh_max)) {
1088 EXT4_ERROR_INODE(inode, "eh_entries %d != eh_max %d!",
1089 path[depth].p_hdr->eh_entries,
1090 path[depth].p_hdr->eh_max);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001091 err = -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001092 goto cleanup;
1093 }
Alex Tomasa86c6182006-10-11 01:21:03 -07001094 /* start copy from next extent */
Yongqiang Yang1b16da72011-05-25 17:41:48 -04001095 m = EXT_MAX_EXTENT(path[depth].p_hdr) - path[depth].p_ext++;
1096 ext4_ext_show_move(inode, path, newblock, depth);
Alex Tomasa86c6182006-10-11 01:21:03 -07001097 if (m) {
Yongqiang Yang1b16da72011-05-25 17:41:48 -04001098 struct ext4_extent *ex;
1099 ex = EXT_FIRST_EXTENT(neh);
1100 memmove(ex, path[depth].p_ext, sizeof(struct ext4_extent) * m);
Marcin Slusarze8546d02008-04-17 10:38:59 -04001101 le16_add_cpu(&neh->eh_entries, m);
Alex Tomasa86c6182006-10-11 01:21:03 -07001102 }
1103
Sriram Rajagopalan592acbf2019-05-10 19:28:06 -04001104 /* zero out unused area in the extent block */
1105 ext_size = sizeof(struct ext4_extent_header) +
1106 sizeof(struct ext4_extent) * le16_to_cpu(neh->eh_entries);
1107 memset(bh->b_data + ext_size, 0, inode->i_sb->s_blocksize - ext_size);
Darrick J. Wong7ac59902012-04-29 18:37:10 -04001108 ext4_extent_block_csum_set(inode, neh);
Alex Tomasa86c6182006-10-11 01:21:03 -07001109 set_buffer_uptodate(bh);
1110 unlock_buffer(bh);
1111
Frank Mayhar03901312009-01-07 00:06:22 -05001112 err = ext4_handle_dirty_metadata(handle, inode, bh);
Avantika Mathur7e028972006-12-06 20:41:33 -08001113 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07001114 goto cleanup;
1115 brelse(bh);
1116 bh = NULL;
1117
1118 /* correct old leaf */
1119 if (m) {
Avantika Mathur7e028972006-12-06 20:41:33 -08001120 err = ext4_ext_get_access(handle, inode, path + depth);
1121 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07001122 goto cleanup;
Marcin Slusarze8546d02008-04-17 10:38:59 -04001123 le16_add_cpu(&path[depth].p_hdr->eh_entries, -m);
Avantika Mathur7e028972006-12-06 20:41:33 -08001124 err = ext4_ext_dirty(handle, inode, path + depth);
1125 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07001126 goto cleanup;
1127
1128 }
1129
1130 /* create intermediate indexes */
1131 k = depth - at - 1;
Frank Mayhar273df552010-03-02 11:46:09 -05001132 if (unlikely(k < 0)) {
1133 EXT4_ERROR_INODE(inode, "k %d < 0!", k);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001134 err = -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001135 goto cleanup;
1136 }
Alex Tomasa86c6182006-10-11 01:21:03 -07001137 if (k)
1138 ext_debug("create %d intermediate indices\n", k);
1139 /* insert new index into current index block */
1140 /* current depth stored in i var */
1141 i = depth - 1;
1142 while (k--) {
1143 oldblock = newblock;
1144 newblock = ablocks[--a];
Eric Sandeenbba90742008-01-28 23:58:27 -05001145 bh = sb_getblk(inode->i_sb, newblock);
Wang Shilongaebf0242013-01-12 16:28:47 -05001146 if (unlikely(!bh)) {
Theodore Ts'o860d21e2013-01-12 16:19:36 -05001147 err = -ENOMEM;
Alex Tomasa86c6182006-10-11 01:21:03 -07001148 goto cleanup;
1149 }
1150 lock_buffer(bh);
1151
Avantika Mathur7e028972006-12-06 20:41:33 -08001152 err = ext4_journal_get_create_access(handle, bh);
1153 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07001154 goto cleanup;
1155
1156 neh = ext_block_hdr(bh);
1157 neh->eh_entries = cpu_to_le16(1);
1158 neh->eh_magic = EXT4_EXT_MAGIC;
Theodore Ts'o55ad63b2009-08-28 10:40:33 -04001159 neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0));
Alex Tomasa86c6182006-10-11 01:21:03 -07001160 neh->eh_depth = cpu_to_le16(depth - i);
1161 fidx = EXT_FIRST_INDEX(neh);
1162 fidx->ei_block = border;
Alex Tomasf65e6fb2006-10-11 01:21:05 -07001163 ext4_idx_store_pblock(fidx, oldblock);
Alex Tomasa86c6182006-10-11 01:21:03 -07001164
Eric Sandeenbba90742008-01-28 23:58:27 -05001165 ext_debug("int.index at %d (block %llu): %u -> %llu\n",
1166 i, newblock, le32_to_cpu(border), oldblock);
Alex Tomasa86c6182006-10-11 01:21:03 -07001167
Yongqiang Yang1b16da72011-05-25 17:41:48 -04001168 /* move remainder of path[i] to the new index block */
Frank Mayhar273df552010-03-02 11:46:09 -05001169 if (unlikely(EXT_MAX_INDEX(path[i].p_hdr) !=
1170 EXT_LAST_INDEX(path[i].p_hdr))) {
1171 EXT4_ERROR_INODE(inode,
1172 "EXT_MAX_INDEX != EXT_LAST_INDEX ee_block %d!",
1173 le32_to_cpu(path[i].p_ext->ee_block));
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001174 err = -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001175 goto cleanup;
1176 }
Yongqiang Yang1b16da72011-05-25 17:41:48 -04001177 /* start copy indexes */
1178 m = EXT_MAX_INDEX(path[i].p_hdr) - path[i].p_idx++;
1179 ext_debug("cur 0x%p, last 0x%p\n", path[i].p_idx,
1180 EXT_MAX_INDEX(path[i].p_hdr));
1181 ext4_ext_show_move(inode, path, newblock, i);
Alex Tomasa86c6182006-10-11 01:21:03 -07001182 if (m) {
Yongqiang Yang1b16da72011-05-25 17:41:48 -04001183 memmove(++fidx, path[i].p_idx,
Alex Tomasa86c6182006-10-11 01:21:03 -07001184 sizeof(struct ext4_extent_idx) * m);
Marcin Slusarze8546d02008-04-17 10:38:59 -04001185 le16_add_cpu(&neh->eh_entries, m);
Alex Tomasa86c6182006-10-11 01:21:03 -07001186 }
Sriram Rajagopalan592acbf2019-05-10 19:28:06 -04001187 /* zero out unused area in the extent block */
1188 ext_size = sizeof(struct ext4_extent_header) +
1189 (sizeof(struct ext4_extent) * le16_to_cpu(neh->eh_entries));
1190 memset(bh->b_data + ext_size, 0,
1191 inode->i_sb->s_blocksize - ext_size);
Darrick J. Wong7ac59902012-04-29 18:37:10 -04001192 ext4_extent_block_csum_set(inode, neh);
Alex Tomasa86c6182006-10-11 01:21:03 -07001193 set_buffer_uptodate(bh);
1194 unlock_buffer(bh);
1195
Frank Mayhar03901312009-01-07 00:06:22 -05001196 err = ext4_handle_dirty_metadata(handle, inode, bh);
Avantika Mathur7e028972006-12-06 20:41:33 -08001197 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07001198 goto cleanup;
1199 brelse(bh);
1200 bh = NULL;
1201
1202 /* correct old index */
1203 if (m) {
1204 err = ext4_ext_get_access(handle, inode, path + i);
1205 if (err)
1206 goto cleanup;
Marcin Slusarze8546d02008-04-17 10:38:59 -04001207 le16_add_cpu(&path[i].p_hdr->eh_entries, -m);
Alex Tomasa86c6182006-10-11 01:21:03 -07001208 err = ext4_ext_dirty(handle, inode, path + i);
1209 if (err)
1210 goto cleanup;
1211 }
1212
1213 i--;
1214 }
1215
1216 /* insert new index */
Alex Tomasa86c6182006-10-11 01:21:03 -07001217 err = ext4_ext_insert_index(handle, inode, path + at,
1218 le32_to_cpu(border), newblock);
1219
1220cleanup:
1221 if (bh) {
1222 if (buffer_locked(bh))
1223 unlock_buffer(bh);
1224 brelse(bh);
1225 }
1226
1227 if (err) {
1228 /* free all allocated blocks in error case */
1229 for (i = 0; i < depth; i++) {
1230 if (!ablocks[i])
1231 continue;
Peter Huewe7dc57612011-02-21 21:01:42 -05001232 ext4_free_blocks(handle, inode, NULL, ablocks[i], 1,
Theodore Ts'oe6362602009-11-23 07:17:05 -05001233 EXT4_FREE_BLOCKS_METADATA);
Alex Tomasa86c6182006-10-11 01:21:03 -07001234 }
1235 }
1236 kfree(ablocks);
1237
1238 return err;
1239}
1240
1241/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001242 * ext4_ext_grow_indepth:
1243 * implements tree growing procedure:
1244 * - allocates new block
1245 * - moves top-level data (index block or leaf) into the new block
1246 * - initializes new top-level, creating index that points to the
1247 * just created block
Alex Tomasa86c6182006-10-11 01:21:03 -07001248 */
1249static int ext4_ext_grow_indepth(handle_t *handle, struct inode *inode,
Dmitry Monakhovbe5cd902014-10-01 22:57:09 -04001250 unsigned int flags)
Alex Tomasa86c6182006-10-11 01:21:03 -07001251{
Alex Tomasa86c6182006-10-11 01:21:03 -07001252 struct ext4_extent_header *neh;
Alex Tomasa86c6182006-10-11 01:21:03 -07001253 struct buffer_head *bh;
Dmitry Monakhovbe5cd902014-10-01 22:57:09 -04001254 ext4_fsblk_t newblock, goal = 0;
1255 struct ext4_super_block *es = EXT4_SB(inode->i_sb)->s_es;
Alex Tomasa86c6182006-10-11 01:21:03 -07001256 int err = 0;
Sriram Rajagopalan592acbf2019-05-10 19:28:06 -04001257 size_t ext_size = 0;
Alex Tomasa86c6182006-10-11 01:21:03 -07001258
Dmitry Monakhovbe5cd902014-10-01 22:57:09 -04001259 /* Try to prepend new index to old one */
1260 if (ext_depth(inode))
1261 goal = ext4_idx_pblock(EXT_FIRST_INDEX(ext_inode_hdr(inode)));
1262 if (goal > le32_to_cpu(es->s_first_data_block)) {
1263 flags |= EXT4_MB_HINT_TRY_GOAL;
1264 goal--;
1265 } else
1266 goal = ext4_inode_to_goal_block(inode);
1267 newblock = ext4_new_meta_blocks(handle, inode, goal, flags,
1268 NULL, &err);
Alex Tomasa86c6182006-10-11 01:21:03 -07001269 if (newblock == 0)
1270 return err;
1271
Nikolay Borisovc45653c2015-07-02 01:34:07 -04001272 bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS);
Wang Shilongaebf0242013-01-12 16:28:47 -05001273 if (unlikely(!bh))
Theodore Ts'o860d21e2013-01-12 16:19:36 -05001274 return -ENOMEM;
Alex Tomasa86c6182006-10-11 01:21:03 -07001275 lock_buffer(bh);
1276
Avantika Mathur7e028972006-12-06 20:41:33 -08001277 err = ext4_journal_get_create_access(handle, bh);
1278 if (err) {
Alex Tomasa86c6182006-10-11 01:21:03 -07001279 unlock_buffer(bh);
1280 goto out;
1281 }
1282
Sriram Rajagopalan592acbf2019-05-10 19:28:06 -04001283 ext_size = sizeof(EXT4_I(inode)->i_data);
Alex Tomasa86c6182006-10-11 01:21:03 -07001284 /* move top-level index/leaf into new block */
Sriram Rajagopalan592acbf2019-05-10 19:28:06 -04001285 memmove(bh->b_data, EXT4_I(inode)->i_data, ext_size);
1286 /* zero out unused area in the extent block */
1287 memset(bh->b_data + ext_size, 0, inode->i_sb->s_blocksize - ext_size);
Alex Tomasa86c6182006-10-11 01:21:03 -07001288
1289 /* set size of new block */
1290 neh = ext_block_hdr(bh);
1291 /* old root could have indexes or leaves
1292 * so calculate e_max right way */
1293 if (ext_depth(inode))
Theodore Ts'o55ad63b2009-08-28 10:40:33 -04001294 neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0));
Alex Tomasa86c6182006-10-11 01:21:03 -07001295 else
Theodore Ts'o55ad63b2009-08-28 10:40:33 -04001296 neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0));
Alex Tomasa86c6182006-10-11 01:21:03 -07001297 neh->eh_magic = EXT4_EXT_MAGIC;
Darrick J. Wong7ac59902012-04-29 18:37:10 -04001298 ext4_extent_block_csum_set(inode, neh);
Alex Tomasa86c6182006-10-11 01:21:03 -07001299 set_buffer_uptodate(bh);
1300 unlock_buffer(bh);
1301
Frank Mayhar03901312009-01-07 00:06:22 -05001302 err = ext4_handle_dirty_metadata(handle, inode, bh);
Avantika Mathur7e028972006-12-06 20:41:33 -08001303 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07001304 goto out;
1305
Dmitry Monakhov1939dd82011-10-22 01:26:05 -04001306 /* Update top-level index: num,max,pointer */
Alex Tomasa86c6182006-10-11 01:21:03 -07001307 neh = ext_inode_hdr(inode);
Dmitry Monakhov1939dd82011-10-22 01:26:05 -04001308 neh->eh_entries = cpu_to_le16(1);
1309 ext4_idx_store_pblock(EXT_FIRST_INDEX(neh), newblock);
1310 if (neh->eh_depth == 0) {
1311 /* Root extent block becomes index block */
1312 neh->eh_max = cpu_to_le16(ext4_ext_space_root_idx(inode, 0));
1313 EXT_FIRST_INDEX(neh)->ei_block =
1314 EXT_FIRST_EXTENT(neh)->ee_block;
1315 }
Mingming Cao2ae02102006-10-11 01:21:11 -07001316 ext_debug("new root: num %d(%d), lblock %d, ptr %llu\n",
Alex Tomasa86c6182006-10-11 01:21:03 -07001317 le16_to_cpu(neh->eh_entries), le16_to_cpu(neh->eh_max),
Andi Kleen5a0790c2010-06-14 13:28:03 -04001318 le32_to_cpu(EXT_FIRST_INDEX(neh)->ei_block),
Theodore Ts'obf89d162010-10-27 21:30:14 -04001319 ext4_idx_pblock(EXT_FIRST_INDEX(neh)));
Alex Tomasa86c6182006-10-11 01:21:03 -07001320
Wei Yongjunba39ebb2012-09-27 09:37:53 -04001321 le16_add_cpu(&neh->eh_depth, 1);
Dmitry Monakhov1939dd82011-10-22 01:26:05 -04001322 ext4_mark_inode_dirty(handle, inode);
Alex Tomasa86c6182006-10-11 01:21:03 -07001323out:
1324 brelse(bh);
1325
1326 return err;
1327}
1328
1329/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001330 * ext4_ext_create_new_leaf:
1331 * finds empty index and adds new leaf.
1332 * if no free index is found, then it requests in-depth growing.
Alex Tomasa86c6182006-10-11 01:21:03 -07001333 */
1334static int ext4_ext_create_new_leaf(handle_t *handle, struct inode *inode,
Theodore Ts'o107a7bd2013-08-16 21:23:41 -04001335 unsigned int mb_flags,
1336 unsigned int gb_flags,
Theodore Ts'odfe50802014-09-01 14:37:09 -04001337 struct ext4_ext_path **ppath,
Allison Henderson55f020d2011-05-25 07:41:26 -04001338 struct ext4_extent *newext)
Alex Tomasa86c6182006-10-11 01:21:03 -07001339{
Theodore Ts'odfe50802014-09-01 14:37:09 -04001340 struct ext4_ext_path *path = *ppath;
Alex Tomasa86c6182006-10-11 01:21:03 -07001341 struct ext4_ext_path *curp;
1342 int depth, i, err = 0;
1343
1344repeat:
1345 i = depth = ext_depth(inode);
1346
1347 /* walk up to the tree and look for free index entry */
1348 curp = path + depth;
1349 while (i > 0 && !EXT_HAS_FREE_INDEX(curp)) {
1350 i--;
1351 curp--;
1352 }
1353
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001354 /* we use already allocated block for index block,
1355 * so subsequent data blocks should be contiguous */
Alex Tomasa86c6182006-10-11 01:21:03 -07001356 if (EXT_HAS_FREE_INDEX(curp)) {
1357 /* if we found index with free entry, then use that
1358 * entry: create all needed subtree and add new leaf */
Theodore Ts'o107a7bd2013-08-16 21:23:41 -04001359 err = ext4_ext_split(handle, inode, mb_flags, path, newext, i);
Shen Feng787e0982008-07-11 19:27:31 -04001360 if (err)
1361 goto out;
Alex Tomasa86c6182006-10-11 01:21:03 -07001362
1363 /* refill path */
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04001364 path = ext4_find_extent(inode,
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -05001365 (ext4_lblk_t)le32_to_cpu(newext->ee_block),
Theodore Ts'odfe50802014-09-01 14:37:09 -04001366 ppath, gb_flags);
Alex Tomasa86c6182006-10-11 01:21:03 -07001367 if (IS_ERR(path))
1368 err = PTR_ERR(path);
1369 } else {
1370 /* tree is full, time to grow in depth */
Dmitry Monakhovbe5cd902014-10-01 22:57:09 -04001371 err = ext4_ext_grow_indepth(handle, inode, mb_flags);
Alex Tomasa86c6182006-10-11 01:21:03 -07001372 if (err)
1373 goto out;
1374
1375 /* refill path */
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04001376 path = ext4_find_extent(inode,
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -05001377 (ext4_lblk_t)le32_to_cpu(newext->ee_block),
Theodore Ts'odfe50802014-09-01 14:37:09 -04001378 ppath, gb_flags);
Alex Tomasa86c6182006-10-11 01:21:03 -07001379 if (IS_ERR(path)) {
1380 err = PTR_ERR(path);
1381 goto out;
1382 }
1383
1384 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001385 * only first (depth 0 -> 1) produces free space;
1386 * in all other cases we have to split the grown tree
Alex Tomasa86c6182006-10-11 01:21:03 -07001387 */
1388 depth = ext_depth(inode);
1389 if (path[depth].p_hdr->eh_entries == path[depth].p_hdr->eh_max) {
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001390 /* now we need to split */
Alex Tomasa86c6182006-10-11 01:21:03 -07001391 goto repeat;
1392 }
1393 }
1394
1395out:
1396 return err;
1397}
1398
1399/*
Alex Tomas1988b512008-01-28 23:58:27 -05001400 * search the closest allocated block to the left for *logical
1401 * and returns it at @logical + it's physical address at @phys
1402 * if *logical is the smallest allocated block, the function
1403 * returns 0 at @phys
1404 * return value contains 0 (success) or error code
1405 */
Theodore Ts'o1f109d52010-10-27 21:30:14 -04001406static int ext4_ext_search_left(struct inode *inode,
1407 struct ext4_ext_path *path,
1408 ext4_lblk_t *logical, ext4_fsblk_t *phys)
Alex Tomas1988b512008-01-28 23:58:27 -05001409{
1410 struct ext4_extent_idx *ix;
1411 struct ext4_extent *ex;
Aneesh Kumar K.Vb939e372008-01-28 23:58:27 -05001412 int depth, ee_len;
Alex Tomas1988b512008-01-28 23:58:27 -05001413
Frank Mayhar273df552010-03-02 11:46:09 -05001414 if (unlikely(path == NULL)) {
1415 EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001416 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001417 }
Alex Tomas1988b512008-01-28 23:58:27 -05001418 depth = path->p_depth;
1419 *phys = 0;
1420
1421 if (depth == 0 && path->p_ext == NULL)
1422 return 0;
1423
1424 /* usually extent in the path covers blocks smaller
1425 * then *logical, but it can be that extent is the
1426 * first one in the file */
1427
1428 ex = path[depth].p_ext;
Aneesh Kumar K.Vb939e372008-01-28 23:58:27 -05001429 ee_len = ext4_ext_get_actual_len(ex);
Alex Tomas1988b512008-01-28 23:58:27 -05001430 if (*logical < le32_to_cpu(ex->ee_block)) {
Frank Mayhar273df552010-03-02 11:46:09 -05001431 if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) {
1432 EXT4_ERROR_INODE(inode,
1433 "EXT_FIRST_EXTENT != ex *logical %d ee_block %d!",
1434 *logical, le32_to_cpu(ex->ee_block));
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001435 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001436 }
Alex Tomas1988b512008-01-28 23:58:27 -05001437 while (--depth >= 0) {
1438 ix = path[depth].p_idx;
Frank Mayhar273df552010-03-02 11:46:09 -05001439 if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) {
1440 EXT4_ERROR_INODE(inode,
1441 "ix (%d) != EXT_FIRST_INDEX (%d) (depth %d)!",
Tao Ma6ee3b212011-10-08 16:08:34 -04001442 ix != NULL ? le32_to_cpu(ix->ei_block) : 0,
Frank Mayhar273df552010-03-02 11:46:09 -05001443 EXT_FIRST_INDEX(path[depth].p_hdr) != NULL ?
Tao Ma6ee3b212011-10-08 16:08:34 -04001444 le32_to_cpu(EXT_FIRST_INDEX(path[depth].p_hdr)->ei_block) : 0,
Frank Mayhar273df552010-03-02 11:46:09 -05001445 depth);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001446 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001447 }
Alex Tomas1988b512008-01-28 23:58:27 -05001448 }
1449 return 0;
1450 }
1451
Frank Mayhar273df552010-03-02 11:46:09 -05001452 if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) {
1453 EXT4_ERROR_INODE(inode,
1454 "logical %d < ee_block %d + ee_len %d!",
1455 *logical, le32_to_cpu(ex->ee_block), ee_len);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001456 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001457 }
Alex Tomas1988b512008-01-28 23:58:27 -05001458
Aneesh Kumar K.Vb939e372008-01-28 23:58:27 -05001459 *logical = le32_to_cpu(ex->ee_block) + ee_len - 1;
Theodore Ts'obf89d162010-10-27 21:30:14 -04001460 *phys = ext4_ext_pblock(ex) + ee_len - 1;
Alex Tomas1988b512008-01-28 23:58:27 -05001461 return 0;
1462}
1463
1464/*
1465 * search the closest allocated block to the right for *logical
1466 * and returns it at @logical + it's physical address at @phys
Tao Madf3ab172011-10-08 15:53:49 -04001467 * if *logical is the largest allocated block, the function
Alex Tomas1988b512008-01-28 23:58:27 -05001468 * returns 0 at @phys
1469 * return value contains 0 (success) or error code
1470 */
Theodore Ts'o1f109d52010-10-27 21:30:14 -04001471static int ext4_ext_search_right(struct inode *inode,
1472 struct ext4_ext_path *path,
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04001473 ext4_lblk_t *logical, ext4_fsblk_t *phys,
1474 struct ext4_extent **ret_ex)
Alex Tomas1988b512008-01-28 23:58:27 -05001475{
1476 struct buffer_head *bh = NULL;
1477 struct ext4_extent_header *eh;
1478 struct ext4_extent_idx *ix;
1479 struct ext4_extent *ex;
1480 ext4_fsblk_t block;
Eric Sandeen395a87b2009-03-10 18:18:47 -04001481 int depth; /* Note, NOT eh_depth; depth from top of tree */
1482 int ee_len;
Alex Tomas1988b512008-01-28 23:58:27 -05001483
Frank Mayhar273df552010-03-02 11:46:09 -05001484 if (unlikely(path == NULL)) {
1485 EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001486 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001487 }
Alex Tomas1988b512008-01-28 23:58:27 -05001488 depth = path->p_depth;
1489 *phys = 0;
1490
1491 if (depth == 0 && path->p_ext == NULL)
1492 return 0;
1493
1494 /* usually extent in the path covers blocks smaller
1495 * then *logical, but it can be that extent is the
1496 * first one in the file */
1497
1498 ex = path[depth].p_ext;
Aneesh Kumar K.Vb939e372008-01-28 23:58:27 -05001499 ee_len = ext4_ext_get_actual_len(ex);
Alex Tomas1988b512008-01-28 23:58:27 -05001500 if (*logical < le32_to_cpu(ex->ee_block)) {
Frank Mayhar273df552010-03-02 11:46:09 -05001501 if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) {
1502 EXT4_ERROR_INODE(inode,
1503 "first_extent(path[%d].p_hdr) != ex",
1504 depth);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001505 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001506 }
Alex Tomas1988b512008-01-28 23:58:27 -05001507 while (--depth >= 0) {
1508 ix = path[depth].p_idx;
Frank Mayhar273df552010-03-02 11:46:09 -05001509 if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) {
1510 EXT4_ERROR_INODE(inode,
1511 "ix != EXT_FIRST_INDEX *logical %d!",
1512 *logical);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001513 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001514 }
Alex Tomas1988b512008-01-28 23:58:27 -05001515 }
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04001516 goto found_extent;
Alex Tomas1988b512008-01-28 23:58:27 -05001517 }
1518
Frank Mayhar273df552010-03-02 11:46:09 -05001519 if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) {
1520 EXT4_ERROR_INODE(inode,
1521 "logical %d < ee_block %d + ee_len %d!",
1522 *logical, le32_to_cpu(ex->ee_block), ee_len);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001523 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001524 }
Alex Tomas1988b512008-01-28 23:58:27 -05001525
1526 if (ex != EXT_LAST_EXTENT(path[depth].p_hdr)) {
1527 /* next allocated block in this leaf */
1528 ex++;
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04001529 goto found_extent;
Alex Tomas1988b512008-01-28 23:58:27 -05001530 }
1531
1532 /* go up and search for index to the right */
1533 while (--depth >= 0) {
1534 ix = path[depth].p_idx;
1535 if (ix != EXT_LAST_INDEX(path[depth].p_hdr))
Wu Fengguang25f1ee32008-11-25 17:24:23 -05001536 goto got_index;
Alex Tomas1988b512008-01-28 23:58:27 -05001537 }
1538
Wu Fengguang25f1ee32008-11-25 17:24:23 -05001539 /* we've gone up to the root and found no index to the right */
1540 return 0;
Alex Tomas1988b512008-01-28 23:58:27 -05001541
Wu Fengguang25f1ee32008-11-25 17:24:23 -05001542got_index:
Alex Tomas1988b512008-01-28 23:58:27 -05001543 /* we've found index to the right, let's
1544 * follow it and find the closest allocated
1545 * block to the right */
1546 ix++;
Theodore Ts'obf89d162010-10-27 21:30:14 -04001547 block = ext4_idx_pblock(ix);
Alex Tomas1988b512008-01-28 23:58:27 -05001548 while (++depth < path->p_depth) {
Eric Sandeen395a87b2009-03-10 18:18:47 -04001549 /* subtract from p_depth to get proper eh_depth */
Theodore Ts'o7d7ea892013-08-16 21:20:41 -04001550 bh = read_extent_tree_block(inode, block,
Theodore Ts'o107a7bd2013-08-16 21:23:41 -04001551 path->p_depth - depth, 0);
Theodore Ts'o7d7ea892013-08-16 21:20:41 -04001552 if (IS_ERR(bh))
1553 return PTR_ERR(bh);
1554 eh = ext_block_hdr(bh);
Alex Tomas1988b512008-01-28 23:58:27 -05001555 ix = EXT_FIRST_INDEX(eh);
Theodore Ts'obf89d162010-10-27 21:30:14 -04001556 block = ext4_idx_pblock(ix);
Alex Tomas1988b512008-01-28 23:58:27 -05001557 put_bh(bh);
1558 }
1559
Theodore Ts'o107a7bd2013-08-16 21:23:41 -04001560 bh = read_extent_tree_block(inode, block, path->p_depth - depth, 0);
Theodore Ts'o7d7ea892013-08-16 21:20:41 -04001561 if (IS_ERR(bh))
1562 return PTR_ERR(bh);
Alex Tomas1988b512008-01-28 23:58:27 -05001563 eh = ext_block_hdr(bh);
Alex Tomas1988b512008-01-28 23:58:27 -05001564 ex = EXT_FIRST_EXTENT(eh);
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04001565found_extent:
Alex Tomas1988b512008-01-28 23:58:27 -05001566 *logical = le32_to_cpu(ex->ee_block);
Theodore Ts'obf89d162010-10-27 21:30:14 -04001567 *phys = ext4_ext_pblock(ex);
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04001568 *ret_ex = ex;
1569 if (bh)
1570 put_bh(bh);
Alex Tomas1988b512008-01-28 23:58:27 -05001571 return 0;
Alex Tomas1988b512008-01-28 23:58:27 -05001572}
1573
1574/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001575 * ext4_ext_next_allocated_block:
Lukas Czernerf17722f2011-06-06 00:05:17 -04001576 * returns allocated block in subsequent extent or EXT_MAX_BLOCKS.
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001577 * NOTE: it considers block number from index entry as
1578 * allocated block. Thus, index entries have to be consistent
1579 * with leaves.
Alex Tomasa86c6182006-10-11 01:21:03 -07001580 */
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04001581ext4_lblk_t
Alex Tomasa86c6182006-10-11 01:21:03 -07001582ext4_ext_next_allocated_block(struct ext4_ext_path *path)
1583{
1584 int depth;
1585
1586 BUG_ON(path == NULL);
1587 depth = path->p_depth;
1588
1589 if (depth == 0 && path->p_ext == NULL)
Lukas Czernerf17722f2011-06-06 00:05:17 -04001590 return EXT_MAX_BLOCKS;
Alex Tomasa86c6182006-10-11 01:21:03 -07001591
1592 while (depth >= 0) {
1593 if (depth == path->p_depth) {
1594 /* leaf */
Curt Wohlgemuth6f8ff532011-10-26 04:38:59 -04001595 if (path[depth].p_ext &&
1596 path[depth].p_ext !=
Alex Tomasa86c6182006-10-11 01:21:03 -07001597 EXT_LAST_EXTENT(path[depth].p_hdr))
1598 return le32_to_cpu(path[depth].p_ext[1].ee_block);
1599 } else {
1600 /* index */
1601 if (path[depth].p_idx !=
1602 EXT_LAST_INDEX(path[depth].p_hdr))
1603 return le32_to_cpu(path[depth].p_idx[1].ei_block);
1604 }
1605 depth--;
1606 }
1607
Lukas Czernerf17722f2011-06-06 00:05:17 -04001608 return EXT_MAX_BLOCKS;
Alex Tomasa86c6182006-10-11 01:21:03 -07001609}
1610
1611/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001612 * ext4_ext_next_leaf_block:
Lukas Czernerf17722f2011-06-06 00:05:17 -04001613 * returns first allocated block from next leaf or EXT_MAX_BLOCKS
Alex Tomasa86c6182006-10-11 01:21:03 -07001614 */
Robin Dong57187892011-07-23 21:49:07 -04001615static ext4_lblk_t ext4_ext_next_leaf_block(struct ext4_ext_path *path)
Alex Tomasa86c6182006-10-11 01:21:03 -07001616{
1617 int depth;
1618
1619 BUG_ON(path == NULL);
1620 depth = path->p_depth;
1621
1622 /* zero-tree has no leaf blocks at all */
1623 if (depth == 0)
Lukas Czernerf17722f2011-06-06 00:05:17 -04001624 return EXT_MAX_BLOCKS;
Alex Tomasa86c6182006-10-11 01:21:03 -07001625
1626 /* go to index block */
1627 depth--;
1628
1629 while (depth >= 0) {
1630 if (path[depth].p_idx !=
1631 EXT_LAST_INDEX(path[depth].p_hdr))
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -05001632 return (ext4_lblk_t)
1633 le32_to_cpu(path[depth].p_idx[1].ei_block);
Alex Tomasa86c6182006-10-11 01:21:03 -07001634 depth--;
1635 }
1636
Lukas Czernerf17722f2011-06-06 00:05:17 -04001637 return EXT_MAX_BLOCKS;
Alex Tomasa86c6182006-10-11 01:21:03 -07001638}
1639
1640/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001641 * ext4_ext_correct_indexes:
1642 * if leaf gets modified and modified extent is first in the leaf,
1643 * then we have to correct all indexes above.
Alex Tomasa86c6182006-10-11 01:21:03 -07001644 * TODO: do we need to correct tree in all cases?
1645 */
Aneesh Kumar K.V1d03ec92008-01-28 23:58:27 -05001646static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode,
Alex Tomasa86c6182006-10-11 01:21:03 -07001647 struct ext4_ext_path *path)
1648{
1649 struct ext4_extent_header *eh;
1650 int depth = ext_depth(inode);
1651 struct ext4_extent *ex;
1652 __le32 border;
1653 int k, err = 0;
1654
1655 eh = path[depth].p_hdr;
1656 ex = path[depth].p_ext;
Frank Mayhar273df552010-03-02 11:46:09 -05001657
1658 if (unlikely(ex == NULL || eh == NULL)) {
1659 EXT4_ERROR_INODE(inode,
1660 "ex %p == NULL or eh %p == NULL", ex, eh);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001661 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001662 }
Alex Tomasa86c6182006-10-11 01:21:03 -07001663
1664 if (depth == 0) {
1665 /* there is no tree at all */
1666 return 0;
1667 }
1668
1669 if (ex != EXT_FIRST_EXTENT(eh)) {
1670 /* we correct tree if first leaf got modified only */
1671 return 0;
1672 }
1673
1674 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001675 * TODO: we need correction if border is smaller than current one
Alex Tomasa86c6182006-10-11 01:21:03 -07001676 */
1677 k = depth - 1;
1678 border = path[depth].p_ext->ee_block;
Avantika Mathur7e028972006-12-06 20:41:33 -08001679 err = ext4_ext_get_access(handle, inode, path + k);
1680 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07001681 return err;
1682 path[k].p_idx->ei_block = border;
Avantika Mathur7e028972006-12-06 20:41:33 -08001683 err = ext4_ext_dirty(handle, inode, path + k);
1684 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07001685 return err;
1686
1687 while (k--) {
1688 /* change all left-side indexes */
1689 if (path[k+1].p_idx != EXT_FIRST_INDEX(path[k+1].p_hdr))
1690 break;
Avantika Mathur7e028972006-12-06 20:41:33 -08001691 err = ext4_ext_get_access(handle, inode, path + k);
1692 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07001693 break;
1694 path[k].p_idx->ei_block = border;
Avantika Mathur7e028972006-12-06 20:41:33 -08001695 err = ext4_ext_dirty(handle, inode, path + k);
1696 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07001697 break;
1698 }
1699
1700 return err;
1701}
1702
Eric Biggers43f81672019-12-31 12:04:40 -06001703static int ext4_can_extents_be_merged(struct inode *inode,
1704 struct ext4_extent *ex1,
1705 struct ext4_extent *ex2)
Alex Tomasa86c6182006-10-11 01:21:03 -07001706{
Eric Sandeenda0169b2013-11-04 09:58:26 -05001707 unsigned short ext1_ee_len, ext2_ee_len;
Amit Aroraa2df2a62007-07-17 21:42:41 -04001708
Lukas Czerner556615d2014-04-20 23:45:47 -04001709 if (ext4_ext_is_unwritten(ex1) != ext4_ext_is_unwritten(ex2))
Amit Aroraa2df2a62007-07-17 21:42:41 -04001710 return 0;
1711
1712 ext1_ee_len = ext4_ext_get_actual_len(ex1);
1713 ext2_ee_len = ext4_ext_get_actual_len(ex2);
1714
1715 if (le32_to_cpu(ex1->ee_block) + ext1_ee_len !=
Andrew Morton63f57932006-10-11 01:21:24 -07001716 le32_to_cpu(ex2->ee_block))
Alex Tomasa86c6182006-10-11 01:21:03 -07001717 return 0;
1718
Suparna Bhattacharya471d4012006-10-11 01:21:06 -07001719 /*
1720 * To allow future support for preallocated extents to be added
1721 * as an RO_COMPAT feature, refuse to merge to extents if
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001722 * this can result in the top bit of ee_len being set.
Suparna Bhattacharya471d4012006-10-11 01:21:06 -07001723 */
Eric Sandeenda0169b2013-11-04 09:58:26 -05001724 if (ext1_ee_len + ext2_ee_len > EXT_INIT_MAX_LEN)
Suparna Bhattacharya471d4012006-10-11 01:21:06 -07001725 return 0;
Matthew Bobrowski378f32b2019-11-05 23:02:39 +11001726
Lukas Czerner556615d2014-04-20 23:45:47 -04001727 if (ext4_ext_is_unwritten(ex1) &&
Matthew Bobrowski378f32b2019-11-05 23:02:39 +11001728 ext1_ee_len + ext2_ee_len > EXT_UNWRITTEN_MAX_LEN)
Darrick J. Wonga9b82412014-02-20 21:17:35 -05001729 return 0;
Robert P. J. Daybbf2f9f2007-02-17 19:20:16 +01001730#ifdef AGGRESSIVE_TEST
Aneesh Kumar K.Vb939e372008-01-28 23:58:27 -05001731 if (ext1_ee_len >= 4)
Alex Tomasa86c6182006-10-11 01:21:03 -07001732 return 0;
1733#endif
1734
Theodore Ts'obf89d162010-10-27 21:30:14 -04001735 if (ext4_ext_pblock(ex1) + ext1_ee_len == ext4_ext_pblock(ex2))
Alex Tomasa86c6182006-10-11 01:21:03 -07001736 return 1;
1737 return 0;
1738}
1739
1740/*
Amit Arora56055d32007-07-17 21:42:38 -04001741 * This function tries to merge the "ex" extent to the next extent in the tree.
1742 * It always tries to merge towards right. If you want to merge towards
1743 * left, pass "ex - 1" as argument instead of "ex".
1744 * Returns 0 if the extents (ex and ex+1) were _not_ merged and returns
1745 * 1 if they got merged.
1746 */
Yongqiang Yang197217a2011-05-03 11:45:29 -04001747static int ext4_ext_try_to_merge_right(struct inode *inode,
Theodore Ts'o1f109d52010-10-27 21:30:14 -04001748 struct ext4_ext_path *path,
1749 struct ext4_extent *ex)
Amit Arora56055d32007-07-17 21:42:38 -04001750{
1751 struct ext4_extent_header *eh;
1752 unsigned int depth, len;
Lukas Czerner556615d2014-04-20 23:45:47 -04001753 int merge_done = 0, unwritten;
Amit Arora56055d32007-07-17 21:42:38 -04001754
1755 depth = ext_depth(inode);
1756 BUG_ON(path[depth].p_hdr == NULL);
1757 eh = path[depth].p_hdr;
1758
1759 while (ex < EXT_LAST_EXTENT(eh)) {
1760 if (!ext4_can_extents_be_merged(inode, ex, ex + 1))
1761 break;
1762 /* merge with next extent! */
Lukas Czerner556615d2014-04-20 23:45:47 -04001763 unwritten = ext4_ext_is_unwritten(ex);
Amit Arora56055d32007-07-17 21:42:38 -04001764 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
1765 + ext4_ext_get_actual_len(ex + 1));
Lukas Czerner556615d2014-04-20 23:45:47 -04001766 if (unwritten)
1767 ext4_ext_mark_unwritten(ex);
Amit Arora56055d32007-07-17 21:42:38 -04001768
1769 if (ex + 1 < EXT_LAST_EXTENT(eh)) {
1770 len = (EXT_LAST_EXTENT(eh) - ex - 1)
1771 * sizeof(struct ext4_extent);
1772 memmove(ex + 1, ex + 2, len);
1773 }
Marcin Slusarze8546d02008-04-17 10:38:59 -04001774 le16_add_cpu(&eh->eh_entries, -1);
Amit Arora56055d32007-07-17 21:42:38 -04001775 merge_done = 1;
1776 WARN_ON(eh->eh_entries == 0);
1777 if (!eh->eh_entries)
Theodore Ts'o24676da2010-05-16 21:00:00 -04001778 EXT4_ERROR_INODE(inode, "eh->eh_entries = 0!");
Amit Arora56055d32007-07-17 21:42:38 -04001779 }
1780
1781 return merge_done;
1782}
1783
1784/*
Theodore Ts'oecb94f52012-08-17 09:44:17 -04001785 * This function does a very simple check to see if we can collapse
1786 * an extent tree with a single extent tree leaf block into the inode.
1787 */
1788static void ext4_ext_try_to_merge_up(handle_t *handle,
1789 struct inode *inode,
1790 struct ext4_ext_path *path)
1791{
1792 size_t s;
1793 unsigned max_root = ext4_ext_space_root(inode, 0);
1794 ext4_fsblk_t blk;
1795
1796 if ((path[0].p_depth != 1) ||
1797 (le16_to_cpu(path[0].p_hdr->eh_entries) != 1) ||
1798 (le16_to_cpu(path[1].p_hdr->eh_entries) > max_root))
1799 return;
1800
1801 /*
1802 * We need to modify the block allocation bitmap and the block
1803 * group descriptor to release the extent tree block. If we
1804 * can't get the journal credits, give up.
1805 */
Jan Kara83448bd2019-11-05 17:44:29 +01001806 if (ext4_journal_extend(handle, 2,
1807 ext4_free_metadata_revoke_credits(inode->i_sb, 1)))
Theodore Ts'oecb94f52012-08-17 09:44:17 -04001808 return;
1809
1810 /*
1811 * Copy the extent data up to the inode
1812 */
1813 blk = ext4_idx_pblock(path[0].p_idx);
1814 s = le16_to_cpu(path[1].p_hdr->eh_entries) *
1815 sizeof(struct ext4_extent_idx);
1816 s += sizeof(struct ext4_extent_header);
1817
Theodore Ts'o10809df82014-09-01 14:40:09 -04001818 path[1].p_maxdepth = path[0].p_maxdepth;
Theodore Ts'oecb94f52012-08-17 09:44:17 -04001819 memcpy(path[0].p_hdr, path[1].p_hdr, s);
1820 path[0].p_depth = 0;
1821 path[0].p_ext = EXT_FIRST_EXTENT(path[0].p_hdr) +
1822 (path[1].p_ext - EXT_FIRST_EXTENT(path[1].p_hdr));
1823 path[0].p_hdr->eh_max = cpu_to_le16(max_root);
1824
1825 brelse(path[1].p_bh);
1826 ext4_free_blocks(handle, inode, NULL, blk, 1,
Theodore Ts'o71d4f7d2014-07-15 06:02:38 -04001827 EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET);
Theodore Ts'oecb94f52012-08-17 09:44:17 -04001828}
1829
1830/*
Yongqiang Yang197217a2011-05-03 11:45:29 -04001831 * This function tries to merge the @ex extent to neighbours in the tree.
1832 * return 1 if merge left else 0.
1833 */
Theodore Ts'oecb94f52012-08-17 09:44:17 -04001834static void ext4_ext_try_to_merge(handle_t *handle,
1835 struct inode *inode,
Yongqiang Yang197217a2011-05-03 11:45:29 -04001836 struct ext4_ext_path *path,
1837 struct ext4_extent *ex) {
1838 struct ext4_extent_header *eh;
1839 unsigned int depth;
1840 int merge_done = 0;
Yongqiang Yang197217a2011-05-03 11:45:29 -04001841
1842 depth = ext_depth(inode);
1843 BUG_ON(path[depth].p_hdr == NULL);
1844 eh = path[depth].p_hdr;
1845
1846 if (ex > EXT_FIRST_EXTENT(eh))
1847 merge_done = ext4_ext_try_to_merge_right(inode, path, ex - 1);
1848
1849 if (!merge_done)
Theodore Ts'oecb94f52012-08-17 09:44:17 -04001850 (void) ext4_ext_try_to_merge_right(inode, path, ex);
Yongqiang Yang197217a2011-05-03 11:45:29 -04001851
Theodore Ts'oecb94f52012-08-17 09:44:17 -04001852 ext4_ext_try_to_merge_up(handle, inode, path);
Yongqiang Yang197217a2011-05-03 11:45:29 -04001853}
1854
1855/*
Amit Arora25d14f92007-05-24 13:04:13 -04001856 * check if a portion of the "newext" extent overlaps with an
1857 * existing extent.
1858 *
1859 * If there is an overlap discovered, it updates the length of the newext
1860 * such that there will be no overlap, and then returns 1.
1861 * If there is no overlap found, it returns 0.
1862 */
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04001863static unsigned int ext4_ext_check_overlap(struct ext4_sb_info *sbi,
1864 struct inode *inode,
Theodore Ts'o1f109d52010-10-27 21:30:14 -04001865 struct ext4_extent *newext,
1866 struct ext4_ext_path *path)
Amit Arora25d14f92007-05-24 13:04:13 -04001867{
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -05001868 ext4_lblk_t b1, b2;
Amit Arora25d14f92007-05-24 13:04:13 -04001869 unsigned int depth, len1;
1870 unsigned int ret = 0;
1871
1872 b1 = le32_to_cpu(newext->ee_block);
Amit Aroraa2df2a62007-07-17 21:42:41 -04001873 len1 = ext4_ext_get_actual_len(newext);
Amit Arora25d14f92007-05-24 13:04:13 -04001874 depth = ext_depth(inode);
1875 if (!path[depth].p_ext)
1876 goto out;
Theodore Ts'of5a44db2013-12-20 09:29:35 -05001877 b2 = EXT4_LBLK_CMASK(sbi, le32_to_cpu(path[depth].p_ext->ee_block));
Amit Arora25d14f92007-05-24 13:04:13 -04001878
1879 /*
1880 * get the next allocated block if the extent in the path
Theodore Ts'o2b2d6d02008-07-26 16:15:44 -04001881 * is before the requested block(s)
Amit Arora25d14f92007-05-24 13:04:13 -04001882 */
1883 if (b2 < b1) {
1884 b2 = ext4_ext_next_allocated_block(path);
Lukas Czernerf17722f2011-06-06 00:05:17 -04001885 if (b2 == EXT_MAX_BLOCKS)
Amit Arora25d14f92007-05-24 13:04:13 -04001886 goto out;
Theodore Ts'of5a44db2013-12-20 09:29:35 -05001887 b2 = EXT4_LBLK_CMASK(sbi, b2);
Amit Arora25d14f92007-05-24 13:04:13 -04001888 }
1889
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -05001890 /* check for wrap through zero on extent logical start block*/
Amit Arora25d14f92007-05-24 13:04:13 -04001891 if (b1 + len1 < b1) {
Lukas Czernerf17722f2011-06-06 00:05:17 -04001892 len1 = EXT_MAX_BLOCKS - b1;
Amit Arora25d14f92007-05-24 13:04:13 -04001893 newext->ee_len = cpu_to_le16(len1);
1894 ret = 1;
1895 }
1896
1897 /* check for overlap */
1898 if (b1 + len1 > b2) {
1899 newext->ee_len = cpu_to_le16(b2 - b1);
1900 ret = 1;
1901 }
1902out:
1903 return ret;
1904}
1905
1906/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07001907 * ext4_ext_insert_extent:
1908 * tries to merge requsted extent into the existing extent or
1909 * inserts requested extent as new one into the tree,
1910 * creating new leaf in the no-space case.
Alex Tomasa86c6182006-10-11 01:21:03 -07001911 */
1912int ext4_ext_insert_extent(handle_t *handle, struct inode *inode,
Theodore Ts'odfe50802014-09-01 14:37:09 -04001913 struct ext4_ext_path **ppath,
Theodore Ts'o107a7bd2013-08-16 21:23:41 -04001914 struct ext4_extent *newext, int gb_flags)
Alex Tomasa86c6182006-10-11 01:21:03 -07001915{
Theodore Ts'odfe50802014-09-01 14:37:09 -04001916 struct ext4_ext_path *path = *ppath;
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001917 struct ext4_extent_header *eh;
Alex Tomasa86c6182006-10-11 01:21:03 -07001918 struct ext4_extent *ex, *fex;
1919 struct ext4_extent *nearex; /* nearest extent */
1920 struct ext4_ext_path *npath = NULL;
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -05001921 int depth, len, err;
1922 ext4_lblk_t next;
Lukas Czerner556615d2014-04-20 23:45:47 -04001923 int mb_flags = 0, unwritten;
Alex Tomasa86c6182006-10-11 01:21:03 -07001924
Theodore Ts'oe3cf5d52014-09-04 18:07:25 -04001925 if (gb_flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
1926 mb_flags |= EXT4_MB_DELALLOC_RESERVED;
Frank Mayhar273df552010-03-02 11:46:09 -05001927 if (unlikely(ext4_ext_get_actual_len(newext) == 0)) {
1928 EXT4_ERROR_INODE(inode, "ext4_ext_get_actual_len(newext) == 0");
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001929 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001930 }
Alex Tomasa86c6182006-10-11 01:21:03 -07001931 depth = ext_depth(inode);
1932 ex = path[depth].p_ext;
Lukas Czernerbe8981b2013-04-03 23:33:28 -04001933 eh = path[depth].p_hdr;
Frank Mayhar273df552010-03-02 11:46:09 -05001934 if (unlikely(path[depth].p_hdr == NULL)) {
1935 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04001936 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05001937 }
Alex Tomasa86c6182006-10-11 01:21:03 -07001938
1939 /* try to insert block into found extent and return */
Theodore Ts'o107a7bd2013-08-16 21:23:41 -04001940 if (ex && !(gb_flags & EXT4_GET_BLOCKS_PRE_IO)) {
Amit Aroraa2df2a62007-07-17 21:42:41 -04001941
1942 /*
Lukas Czernerbe8981b2013-04-03 23:33:28 -04001943 * Try to see whether we should rather test the extent on
1944 * right from ex, or from the left of ex. This is because
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04001945 * ext4_find_extent() can return either extent on the
Lukas Czernerbe8981b2013-04-03 23:33:28 -04001946 * left, or on the right from the searched position. This
1947 * will make merging more effective.
Amit Aroraa2df2a62007-07-17 21:42:41 -04001948 */
Lukas Czernerbe8981b2013-04-03 23:33:28 -04001949 if (ex < EXT_LAST_EXTENT(eh) &&
1950 (le32_to_cpu(ex->ee_block) +
1951 ext4_ext_get_actual_len(ex) <
1952 le32_to_cpu(newext->ee_block))) {
1953 ex += 1;
1954 goto prepend;
1955 } else if ((ex > EXT_FIRST_EXTENT(eh)) &&
1956 (le32_to_cpu(newext->ee_block) +
1957 ext4_ext_get_actual_len(newext) <
1958 le32_to_cpu(ex->ee_block)))
1959 ex -= 1;
1960
1961 /* Try to append newex to the ex */
1962 if (ext4_can_extents_be_merged(inode, ex, newext)) {
1963 ext_debug("append [%d]%d block to %u:[%d]%d"
1964 "(from %llu)\n",
Lukas Czerner556615d2014-04-20 23:45:47 -04001965 ext4_ext_is_unwritten(newext),
Lukas Czernerbe8981b2013-04-03 23:33:28 -04001966 ext4_ext_get_actual_len(newext),
1967 le32_to_cpu(ex->ee_block),
Lukas Czerner556615d2014-04-20 23:45:47 -04001968 ext4_ext_is_unwritten(ex),
Lukas Czernerbe8981b2013-04-03 23:33:28 -04001969 ext4_ext_get_actual_len(ex),
1970 ext4_ext_pblock(ex));
1971 err = ext4_ext_get_access(handle, inode,
1972 path + depth);
1973 if (err)
1974 return err;
Lukas Czerner556615d2014-04-20 23:45:47 -04001975 unwritten = ext4_ext_is_unwritten(ex);
Lukas Czernerbe8981b2013-04-03 23:33:28 -04001976 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
Amit Aroraa2df2a62007-07-17 21:42:41 -04001977 + ext4_ext_get_actual_len(newext));
Lukas Czerner556615d2014-04-20 23:45:47 -04001978 if (unwritten)
1979 ext4_ext_mark_unwritten(ex);
Lukas Czernerbe8981b2013-04-03 23:33:28 -04001980 eh = path[depth].p_hdr;
1981 nearex = ex;
1982 goto merge;
1983 }
1984
1985prepend:
1986 /* Try to prepend newex to the ex */
1987 if (ext4_can_extents_be_merged(inode, newext, ex)) {
1988 ext_debug("prepend %u[%d]%d block to %u:[%d]%d"
1989 "(from %llu)\n",
1990 le32_to_cpu(newext->ee_block),
Lukas Czerner556615d2014-04-20 23:45:47 -04001991 ext4_ext_is_unwritten(newext),
Lukas Czernerbe8981b2013-04-03 23:33:28 -04001992 ext4_ext_get_actual_len(newext),
1993 le32_to_cpu(ex->ee_block),
Lukas Czerner556615d2014-04-20 23:45:47 -04001994 ext4_ext_is_unwritten(ex),
Lukas Czernerbe8981b2013-04-03 23:33:28 -04001995 ext4_ext_get_actual_len(ex),
1996 ext4_ext_pblock(ex));
1997 err = ext4_ext_get_access(handle, inode,
1998 path + depth);
1999 if (err)
2000 return err;
2001
Lukas Czerner556615d2014-04-20 23:45:47 -04002002 unwritten = ext4_ext_is_unwritten(ex);
Lukas Czernerbe8981b2013-04-03 23:33:28 -04002003 ex->ee_block = newext->ee_block;
2004 ext4_ext_store_pblock(ex, ext4_ext_pblock(newext));
2005 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
2006 + ext4_ext_get_actual_len(newext));
Lukas Czerner556615d2014-04-20 23:45:47 -04002007 if (unwritten)
2008 ext4_ext_mark_unwritten(ex);
Lukas Czernerbe8981b2013-04-03 23:33:28 -04002009 eh = path[depth].p_hdr;
2010 nearex = ex;
2011 goto merge;
2012 }
Alex Tomasa86c6182006-10-11 01:21:03 -07002013 }
2014
Alex Tomasa86c6182006-10-11 01:21:03 -07002015 depth = ext_depth(inode);
2016 eh = path[depth].p_hdr;
2017 if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max))
2018 goto has_space;
2019
2020 /* probably next leaf has space for us? */
2021 fex = EXT_LAST_EXTENT(eh);
Robin Dong598dbdf2011-07-11 18:24:01 -04002022 next = EXT_MAX_BLOCKS;
2023 if (le32_to_cpu(newext->ee_block) > le32_to_cpu(fex->ee_block))
Robin Dong57187892011-07-23 21:49:07 -04002024 next = ext4_ext_next_leaf_block(path);
Robin Dong598dbdf2011-07-11 18:24:01 -04002025 if (next != EXT_MAX_BLOCKS) {
Yongqiang Yang32de6752011-11-01 18:56:41 -04002026 ext_debug("next leaf block - %u\n", next);
Alex Tomasa86c6182006-10-11 01:21:03 -07002027 BUG_ON(npath != NULL);
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04002028 npath = ext4_find_extent(inode, next, NULL, 0);
Alex Tomasa86c6182006-10-11 01:21:03 -07002029 if (IS_ERR(npath))
2030 return PTR_ERR(npath);
2031 BUG_ON(npath->p_depth != path->p_depth);
2032 eh = npath[depth].p_hdr;
2033 if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max)) {
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002034 ext_debug("next leaf isn't full(%d)\n",
Alex Tomasa86c6182006-10-11 01:21:03 -07002035 le16_to_cpu(eh->eh_entries));
2036 path = npath;
Robin Dongffb505f2011-07-11 11:43:59 -04002037 goto has_space;
Alex Tomasa86c6182006-10-11 01:21:03 -07002038 }
2039 ext_debug("next leaf has no free space(%d,%d)\n",
2040 le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max));
2041 }
2042
2043 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07002044 * There is no free space in the found leaf.
2045 * We're gonna add a new leaf in the tree.
Alex Tomasa86c6182006-10-11 01:21:03 -07002046 */
Theodore Ts'o107a7bd2013-08-16 21:23:41 -04002047 if (gb_flags & EXT4_GET_BLOCKS_METADATA_NOFAIL)
Theodore Ts'oe3cf5d52014-09-04 18:07:25 -04002048 mb_flags |= EXT4_MB_USE_RESERVED;
Theodore Ts'o107a7bd2013-08-16 21:23:41 -04002049 err = ext4_ext_create_new_leaf(handle, inode, mb_flags, gb_flags,
Theodore Ts'odfe50802014-09-01 14:37:09 -04002050 ppath, newext);
Alex Tomasa86c6182006-10-11 01:21:03 -07002051 if (err)
2052 goto cleanup;
2053 depth = ext_depth(inode);
2054 eh = path[depth].p_hdr;
2055
2056has_space:
2057 nearex = path[depth].p_ext;
2058
Avantika Mathur7e028972006-12-06 20:41:33 -08002059 err = ext4_ext_get_access(handle, inode, path + depth);
2060 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07002061 goto cleanup;
2062
2063 if (!nearex) {
2064 /* there is no extent in this leaf, create first one */
Yongqiang Yang32de6752011-11-01 18:56:41 -04002065 ext_debug("first extent in the leaf: %u:%llu:[%d]%d\n",
Dave Kleikamp8c55e202007-05-24 13:04:54 -04002066 le32_to_cpu(newext->ee_block),
Theodore Ts'obf89d162010-10-27 21:30:14 -04002067 ext4_ext_pblock(newext),
Lukas Czerner556615d2014-04-20 23:45:47 -04002068 ext4_ext_is_unwritten(newext),
Amit Aroraa2df2a62007-07-17 21:42:41 -04002069 ext4_ext_get_actual_len(newext));
Eric Gouriou80e675f2011-10-27 11:52:18 -04002070 nearex = EXT_FIRST_EXTENT(eh);
2071 } else {
2072 if (le32_to_cpu(newext->ee_block)
Dave Kleikamp8c55e202007-05-24 13:04:54 -04002073 > le32_to_cpu(nearex->ee_block)) {
Eric Gouriou80e675f2011-10-27 11:52:18 -04002074 /* Insert after */
Yongqiang Yang32de6752011-11-01 18:56:41 -04002075 ext_debug("insert %u:%llu:[%d]%d before: "
2076 "nearest %p\n",
Dave Kleikamp8c55e202007-05-24 13:04:54 -04002077 le32_to_cpu(newext->ee_block),
Theodore Ts'obf89d162010-10-27 21:30:14 -04002078 ext4_ext_pblock(newext),
Lukas Czerner556615d2014-04-20 23:45:47 -04002079 ext4_ext_is_unwritten(newext),
Amit Aroraa2df2a62007-07-17 21:42:41 -04002080 ext4_ext_get_actual_len(newext),
Eric Gouriou80e675f2011-10-27 11:52:18 -04002081 nearex);
2082 nearex++;
2083 } else {
2084 /* Insert before */
2085 BUG_ON(newext->ee_block == nearex->ee_block);
Yongqiang Yang32de6752011-11-01 18:56:41 -04002086 ext_debug("insert %u:%llu:[%d]%d after: "
2087 "nearest %p\n",
Eric Gouriou80e675f2011-10-27 11:52:18 -04002088 le32_to_cpu(newext->ee_block),
2089 ext4_ext_pblock(newext),
Lukas Czerner556615d2014-04-20 23:45:47 -04002090 ext4_ext_is_unwritten(newext),
Eric Gouriou80e675f2011-10-27 11:52:18 -04002091 ext4_ext_get_actual_len(newext),
2092 nearex);
Alex Tomasa86c6182006-10-11 01:21:03 -07002093 }
Eric Gouriou80e675f2011-10-27 11:52:18 -04002094 len = EXT_LAST_EXTENT(eh) - nearex + 1;
2095 if (len > 0) {
Yongqiang Yang32de6752011-11-01 18:56:41 -04002096 ext_debug("insert %u:%llu:[%d]%d: "
Eric Gouriou80e675f2011-10-27 11:52:18 -04002097 "move %d extents from 0x%p to 0x%p\n",
2098 le32_to_cpu(newext->ee_block),
2099 ext4_ext_pblock(newext),
Lukas Czerner556615d2014-04-20 23:45:47 -04002100 ext4_ext_is_unwritten(newext),
Eric Gouriou80e675f2011-10-27 11:52:18 -04002101 ext4_ext_get_actual_len(newext),
2102 len, nearex, nearex + 1);
2103 memmove(nearex + 1, nearex,
2104 len * sizeof(struct ext4_extent));
2105 }
Alex Tomasa86c6182006-10-11 01:21:03 -07002106 }
2107
Marcin Slusarze8546d02008-04-17 10:38:59 -04002108 le16_add_cpu(&eh->eh_entries, 1);
Eric Gouriou80e675f2011-10-27 11:52:18 -04002109 path[depth].p_ext = nearex;
Alex Tomasa86c6182006-10-11 01:21:03 -07002110 nearex->ee_block = newext->ee_block;
Theodore Ts'obf89d162010-10-27 21:30:14 -04002111 ext4_ext_store_pblock(nearex, ext4_ext_pblock(newext));
Alex Tomasa86c6182006-10-11 01:21:03 -07002112 nearex->ee_len = newext->ee_len;
Alex Tomasa86c6182006-10-11 01:21:03 -07002113
2114merge:
HaiboLiue7bcf822012-07-09 16:29:28 -04002115 /* try to merge extents */
Theodore Ts'o107a7bd2013-08-16 21:23:41 -04002116 if (!(gb_flags & EXT4_GET_BLOCKS_PRE_IO))
Theodore Ts'oecb94f52012-08-17 09:44:17 -04002117 ext4_ext_try_to_merge(handle, inode, path, nearex);
Alex Tomasa86c6182006-10-11 01:21:03 -07002118
Alex Tomasa86c6182006-10-11 01:21:03 -07002119
2120 /* time to correct all indexes above */
2121 err = ext4_ext_correct_indexes(handle, inode, path);
2122 if (err)
2123 goto cleanup;
2124
Theodore Ts'oecb94f52012-08-17 09:44:17 -04002125 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
Alex Tomasa86c6182006-10-11 01:21:03 -07002126
2127cleanup:
Theodore Ts'ob7ea89a2014-09-01 14:39:09 -04002128 ext4_ext_drop_refs(npath);
2129 kfree(npath);
Alex Tomasa86c6182006-10-11 01:21:03 -07002130 return err;
2131}
2132
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002133static int ext4_fill_fiemap_extents(struct inode *inode,
2134 ext4_lblk_t block, ext4_lblk_t num,
2135 struct fiemap_extent_info *fieinfo)
Eric Sandeen6873fa02008-10-07 00:46:36 -04002136{
2137 struct ext4_ext_path *path = NULL;
Eric Sandeen6873fa02008-10-07 00:46:36 -04002138 struct ext4_extent *ex;
Zheng Liu69eb33d2013-02-18 00:31:07 -05002139 struct extent_status es;
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002140 ext4_lblk_t next, next_del, start = 0, end = 0;
Eric Sandeen6873fa02008-10-07 00:46:36 -04002141 ext4_lblk_t last = block + num;
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002142 int exists, depth = 0, err = 0;
2143 unsigned int flags = 0;
2144 unsigned char blksize_bits = inode->i_sb->s_blocksize_bits;
Eric Sandeen6873fa02008-10-07 00:46:36 -04002145
Lukas Czernerf17722f2011-06-06 00:05:17 -04002146 while (block < last && block != EXT_MAX_BLOCKS) {
Eric Sandeen6873fa02008-10-07 00:46:36 -04002147 num = last - block;
2148 /* find extent for this block */
Theodore Ts'ofab3a542009-12-09 21:30:02 -05002149 down_read(&EXT4_I(inode)->i_data_sem);
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002150
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04002151 path = ext4_find_extent(inode, block, &path, 0);
Eric Sandeen6873fa02008-10-07 00:46:36 -04002152 if (IS_ERR(path)) {
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002153 up_read(&EXT4_I(inode)->i_data_sem);
Eric Sandeen6873fa02008-10-07 00:46:36 -04002154 err = PTR_ERR(path);
2155 path = NULL;
2156 break;
2157 }
2158
2159 depth = ext_depth(inode);
Frank Mayhar273df552010-03-02 11:46:09 -05002160 if (unlikely(path[depth].p_hdr == NULL)) {
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002161 up_read(&EXT4_I(inode)->i_data_sem);
Frank Mayhar273df552010-03-02 11:46:09 -05002162 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04002163 err = -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05002164 break;
2165 }
Eric Sandeen6873fa02008-10-07 00:46:36 -04002166 ex = path[depth].p_ext;
2167 next = ext4_ext_next_allocated_block(path);
2168
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002169 flags = 0;
Eric Sandeen6873fa02008-10-07 00:46:36 -04002170 exists = 0;
2171 if (!ex) {
2172 /* there is no extent yet, so try to allocate
2173 * all requested space */
2174 start = block;
2175 end = block + num;
2176 } else if (le32_to_cpu(ex->ee_block) > block) {
2177 /* need to allocate space before found extent */
2178 start = block;
2179 end = le32_to_cpu(ex->ee_block);
2180 if (block + num < end)
2181 end = block + num;
2182 } else if (block >= le32_to_cpu(ex->ee_block)
2183 + ext4_ext_get_actual_len(ex)) {
2184 /* need to allocate space after found extent */
2185 start = block;
2186 end = block + num;
2187 if (end >= next)
2188 end = next;
2189 } else if (block >= le32_to_cpu(ex->ee_block)) {
2190 /*
2191 * some part of requested space is covered
2192 * by found extent
2193 */
2194 start = block;
2195 end = le32_to_cpu(ex->ee_block)
2196 + ext4_ext_get_actual_len(ex);
2197 if (block + num < end)
2198 end = block + num;
2199 exists = 1;
2200 } else {
2201 BUG();
2202 }
2203 BUG_ON(end <= start);
2204
2205 if (!exists) {
Zheng Liu69eb33d2013-02-18 00:31:07 -05002206 es.es_lblk = start;
2207 es.es_len = end - start;
2208 es.es_pblk = 0;
Eric Sandeen6873fa02008-10-07 00:46:36 -04002209 } else {
Zheng Liu69eb33d2013-02-18 00:31:07 -05002210 es.es_lblk = le32_to_cpu(ex->ee_block);
2211 es.es_len = ext4_ext_get_actual_len(ex);
2212 es.es_pblk = ext4_ext_pblock(ex);
Lukas Czerner556615d2014-04-20 23:45:47 -04002213 if (ext4_ext_is_unwritten(ex))
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002214 flags |= FIEMAP_EXTENT_UNWRITTEN;
Eric Sandeen6873fa02008-10-07 00:46:36 -04002215 }
2216
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002217 /*
Zheng Liu69eb33d2013-02-18 00:31:07 -05002218 * Find delayed extent and update es accordingly. We call
2219 * it even in !exists case to find out whether es is the
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002220 * last existing extent or not.
2221 */
Zheng Liu69eb33d2013-02-18 00:31:07 -05002222 next_del = ext4_find_delayed_extent(inode, &es);
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002223 if (!exists && next_del) {
2224 exists = 1;
Jie Liu72dac952013-06-12 23:13:59 -04002225 flags |= (FIEMAP_EXTENT_DELALLOC |
2226 FIEMAP_EXTENT_UNKNOWN);
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002227 }
2228 up_read(&EXT4_I(inode)->i_data_sem);
2229
Zheng Liu69eb33d2013-02-18 00:31:07 -05002230 if (unlikely(es.es_len == 0)) {
2231 EXT4_ERROR_INODE(inode, "es.es_len == 0");
Darrick J. Wong6a797d22015-10-17 16:16:04 -04002232 err = -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05002233 break;
2234 }
Eric Sandeen6873fa02008-10-07 00:46:36 -04002235
Zheng Liuf7fec032013-02-18 00:28:47 -05002236 /*
2237 * This is possible iff next == next_del == EXT_MAX_BLOCKS.
2238 * we need to check next == EXT_MAX_BLOCKS because it is
2239 * possible that an extent is with unwritten and delayed
2240 * status due to when an extent is delayed allocated and
2241 * is allocated by fallocate status tree will track both of
2242 * them in a extent.
2243 *
2244 * So we could return a unwritten and delayed extent, and
2245 * its block is equal to 'next'.
2246 */
2247 if (next == next_del && next == EXT_MAX_BLOCKS) {
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002248 flags |= FIEMAP_EXTENT_LAST;
2249 if (unlikely(next_del != EXT_MAX_BLOCKS ||
2250 next != EXT_MAX_BLOCKS)) {
2251 EXT4_ERROR_INODE(inode,
2252 "next extent == %u, next "
2253 "delalloc extent = %u",
2254 next, next_del);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04002255 err = -EFSCORRUPTED;
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002256 break;
2257 }
Eric Sandeen6873fa02008-10-07 00:46:36 -04002258 }
2259
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002260 if (exists) {
2261 err = fiemap_fill_next_extent(fieinfo,
Zheng Liu69eb33d2013-02-18 00:31:07 -05002262 (__u64)es.es_lblk << blksize_bits,
2263 (__u64)es.es_pblk << blksize_bits,
2264 (__u64)es.es_len << blksize_bits,
Lukas Czerner91dd8c12012-11-28 12:32:26 -05002265 flags);
2266 if (err < 0)
2267 break;
2268 if (err == 1) {
2269 err = 0;
2270 break;
2271 }
Eric Sandeen6873fa02008-10-07 00:46:36 -04002272 }
2273
Zheng Liu69eb33d2013-02-18 00:31:07 -05002274 block = es.es_lblk + es.es_len;
Eric Sandeen6873fa02008-10-07 00:46:36 -04002275 }
2276
Theodore Ts'ob7ea89a2014-09-01 14:39:09 -04002277 ext4_ext_drop_refs(path);
2278 kfree(path);
Eric Sandeen6873fa02008-10-07 00:46:36 -04002279 return err;
2280}
2281
Theodore Ts'obb5835e2019-08-11 16:32:41 -04002282static int ext4_fill_es_cache_info(struct inode *inode,
2283 ext4_lblk_t block, ext4_lblk_t num,
2284 struct fiemap_extent_info *fieinfo)
2285{
2286 ext4_lblk_t next, end = block + num - 1;
2287 struct extent_status es;
2288 unsigned char blksize_bits = inode->i_sb->s_blocksize_bits;
2289 unsigned int flags;
2290 int err;
2291
2292 while (block <= end) {
2293 next = 0;
2294 flags = 0;
2295 if (!ext4_es_lookup_extent(inode, block, &next, &es))
2296 break;
2297 if (ext4_es_is_unwritten(&es))
2298 flags |= FIEMAP_EXTENT_UNWRITTEN;
2299 if (ext4_es_is_delayed(&es))
2300 flags |= (FIEMAP_EXTENT_DELALLOC |
2301 FIEMAP_EXTENT_UNKNOWN);
2302 if (ext4_es_is_hole(&es))
2303 flags |= EXT4_FIEMAP_EXTENT_HOLE;
2304 if (next == 0)
2305 flags |= FIEMAP_EXTENT_LAST;
2306 if (flags & (FIEMAP_EXTENT_DELALLOC|
2307 EXT4_FIEMAP_EXTENT_HOLE))
2308 es.es_pblk = 0;
2309 else
2310 es.es_pblk = ext4_es_pblock(&es);
2311 err = fiemap_fill_next_extent(fieinfo,
2312 (__u64)es.es_lblk << blksize_bits,
2313 (__u64)es.es_pblk << blksize_bits,
2314 (__u64)es.es_len << blksize_bits,
2315 flags);
2316 if (next == 0)
2317 break;
2318 block = next;
2319 if (err < 0)
2320 return err;
2321 if (err == 1)
2322 return 0;
2323 }
2324 return 0;
2325}
2326
2327
Alex Tomasa86c6182006-10-11 01:21:03 -07002328/*
Jan Kara140a5252016-03-09 22:46:57 -05002329 * ext4_ext_determine_hole - determine hole around given block
2330 * @inode: inode we lookup in
2331 * @path: path in extent tree to @lblk
2332 * @lblk: pointer to logical block around which we want to determine hole
2333 *
2334 * Determine hole length (and start if easily possible) around given logical
2335 * block. We don't try too hard to find the beginning of the hole but @path
2336 * actually points to extent before @lblk, we provide it.
2337 *
2338 * The function returns the length of a hole starting at @lblk. We update @lblk
2339 * to the beginning of the hole if we managed to find it.
2340 */
2341static ext4_lblk_t ext4_ext_determine_hole(struct inode *inode,
2342 struct ext4_ext_path *path,
2343 ext4_lblk_t *lblk)
2344{
2345 int depth = ext_depth(inode);
2346 struct ext4_extent *ex;
2347 ext4_lblk_t len;
2348
2349 ex = path[depth].p_ext;
2350 if (ex == NULL) {
2351 /* there is no extent yet, so gap is [0;-] */
2352 *lblk = 0;
2353 len = EXT_MAX_BLOCKS;
2354 } else if (*lblk < le32_to_cpu(ex->ee_block)) {
2355 len = le32_to_cpu(ex->ee_block) - *lblk;
2356 } else if (*lblk >= le32_to_cpu(ex->ee_block)
2357 + ext4_ext_get_actual_len(ex)) {
2358 ext4_lblk_t next;
2359
2360 *lblk = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex);
2361 next = ext4_ext_next_allocated_block(path);
2362 BUG_ON(next == *lblk);
2363 len = next - *lblk;
2364 } else {
2365 BUG();
2366 }
2367 return len;
2368}
2369
2370/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07002371 * ext4_ext_put_gap_in_cache:
2372 * calculate boundaries of the gap that the requested block fits into
Alex Tomasa86c6182006-10-11 01:21:03 -07002373 * and cache this gap
2374 */
Avantika Mathur09b88252006-12-06 20:41:36 -08002375static void
Jan Kara140a5252016-03-09 22:46:57 -05002376ext4_ext_put_gap_in_cache(struct inode *inode, ext4_lblk_t hole_start,
2377 ext4_lblk_t hole_len)
Alex Tomasa86c6182006-10-11 01:21:03 -07002378{
Zheng Liu2f8e0a72014-11-25 11:44:37 -05002379 struct extent_status es;
Alex Tomasa86c6182006-10-11 01:21:03 -07002380
Eric Whitneyad431022018-10-01 14:10:39 -04002381 ext4_es_find_extent_range(inode, &ext4_es_is_delayed, hole_start,
2382 hole_start + hole_len - 1, &es);
Zheng Liu2f8e0a72014-11-25 11:44:37 -05002383 if (es.es_len) {
2384 /* There's delayed extent containing lblock? */
Jan Kara140a5252016-03-09 22:46:57 -05002385 if (es.es_lblk <= hole_start)
Zheng Liu2f8e0a72014-11-25 11:44:37 -05002386 return;
Jan Kara140a5252016-03-09 22:46:57 -05002387 hole_len = min(es.es_lblk - hole_start, hole_len);
Zheng Liu2f8e0a72014-11-25 11:44:37 -05002388 }
Jan Kara140a5252016-03-09 22:46:57 -05002389 ext_debug(" -> %u:%u\n", hole_start, hole_len);
2390 ext4_es_insert_extent(inode, hole_start, hole_len, ~0,
2391 EXTENT_STATUS_HOLE);
Alex Tomasa86c6182006-10-11 01:21:03 -07002392}
2393
2394/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07002395 * ext4_ext_rm_idx:
2396 * removes index from the index block.
Alex Tomasa86c6182006-10-11 01:21:03 -07002397 */
Aneesh Kumar K.V1d03ec92008-01-28 23:58:27 -05002398static int ext4_ext_rm_idx(handle_t *handle, struct inode *inode,
Forrest Liuc36575e2012-12-17 09:55:39 -05002399 struct ext4_ext_path *path, int depth)
Alex Tomasa86c6182006-10-11 01:21:03 -07002400{
Alex Tomasa86c6182006-10-11 01:21:03 -07002401 int err;
Alex Tomasf65e6fb2006-10-11 01:21:05 -07002402 ext4_fsblk_t leaf;
Alex Tomasa86c6182006-10-11 01:21:03 -07002403
2404 /* free index block */
Forrest Liuc36575e2012-12-17 09:55:39 -05002405 depth--;
2406 path = path + depth;
Theodore Ts'obf89d162010-10-27 21:30:14 -04002407 leaf = ext4_idx_pblock(path->p_idx);
Frank Mayhar273df552010-03-02 11:46:09 -05002408 if (unlikely(path->p_hdr->eh_entries == 0)) {
2409 EXT4_ERROR_INODE(inode, "path->p_hdr->eh_entries == 0");
Darrick J. Wong6a797d22015-10-17 16:16:04 -04002410 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05002411 }
Avantika Mathur7e028972006-12-06 20:41:33 -08002412 err = ext4_ext_get_access(handle, inode, path);
2413 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07002414 return err;
Robin Dong0e1147b2011-07-27 21:29:33 -04002415
2416 if (path->p_idx != EXT_LAST_INDEX(path->p_hdr)) {
2417 int len = EXT_LAST_INDEX(path->p_hdr) - path->p_idx;
2418 len *= sizeof(struct ext4_extent_idx);
2419 memmove(path->p_idx, path->p_idx + 1, len);
2420 }
2421
Marcin Slusarze8546d02008-04-17 10:38:59 -04002422 le16_add_cpu(&path->p_hdr->eh_entries, -1);
Avantika Mathur7e028972006-12-06 20:41:33 -08002423 err = ext4_ext_dirty(handle, inode, path);
2424 if (err)
Alex Tomasa86c6182006-10-11 01:21:03 -07002425 return err;
Mingming Cao2ae02102006-10-11 01:21:11 -07002426 ext_debug("index is empty, remove it, free block %llu\n", leaf);
Aditya Kalid8990242011-09-09 19:18:51 -04002427 trace_ext4_ext_rm_idx(inode, leaf);
2428
Peter Huewe7dc57612011-02-21 21:01:42 -05002429 ext4_free_blocks(handle, inode, NULL, leaf, 1,
Theodore Ts'oe6362602009-11-23 07:17:05 -05002430 EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET);
Forrest Liuc36575e2012-12-17 09:55:39 -05002431
2432 while (--depth >= 0) {
2433 if (path->p_idx != EXT_FIRST_INDEX(path->p_hdr))
2434 break;
2435 path--;
2436 err = ext4_ext_get_access(handle, inode, path);
2437 if (err)
2438 break;
2439 path->p_idx->ei_block = (path+1)->p_idx->ei_block;
2440 err = ext4_ext_dirty(handle, inode, path);
2441 if (err)
2442 break;
2443 }
Alex Tomasa86c6182006-10-11 01:21:03 -07002444 return err;
2445}
2446
2447/*
Mingming Caoee12b632008-08-19 22:16:05 -04002448 * ext4_ext_calc_credits_for_single_extent:
2449 * This routine returns max. credits that needed to insert an extent
2450 * to the extent tree.
2451 * When pass the actual path, the caller should calculate credits
2452 * under i_data_sem.
Alex Tomasa86c6182006-10-11 01:21:03 -07002453 */
Mingming Cao525f4ed2008-08-19 22:15:58 -04002454int ext4_ext_calc_credits_for_single_extent(struct inode *inode, int nrblocks,
Alex Tomasa86c6182006-10-11 01:21:03 -07002455 struct ext4_ext_path *path)
2456{
Alex Tomasa86c6182006-10-11 01:21:03 -07002457 if (path) {
Mingming Caoee12b632008-08-19 22:16:05 -04002458 int depth = ext_depth(inode);
Mingming Caof3bd1f32008-08-19 22:16:03 -04002459 int ret = 0;
Mingming Caoee12b632008-08-19 22:16:05 -04002460
Alex Tomasa86c6182006-10-11 01:21:03 -07002461 /* probably there is space in leaf? */
Alex Tomasa86c6182006-10-11 01:21:03 -07002462 if (le16_to_cpu(path[depth].p_hdr->eh_entries)
Mingming Caoee12b632008-08-19 22:16:05 -04002463 < le16_to_cpu(path[depth].p_hdr->eh_max)) {
2464
2465 /*
2466 * There are some space in the leaf tree, no
2467 * need to account for leaf block credit
2468 *
2469 * bitmaps and block group descriptor blocks
Tao Madf3ab172011-10-08 15:53:49 -04002470 * and other metadata blocks still need to be
Mingming Caoee12b632008-08-19 22:16:05 -04002471 * accounted.
2472 */
Mingming Cao525f4ed2008-08-19 22:15:58 -04002473 /* 1 bitmap, 1 block group descriptor */
Mingming Caoee12b632008-08-19 22:16:05 -04002474 ret = 2 + EXT4_META_TRANS_BLOCKS(inode->i_sb);
Aneesh Kumar K.V5887e982009-07-05 23:12:04 -04002475 return ret;
Mingming Caoee12b632008-08-19 22:16:05 -04002476 }
Alex Tomasa86c6182006-10-11 01:21:03 -07002477 }
2478
Mingming Cao525f4ed2008-08-19 22:15:58 -04002479 return ext4_chunk_trans_blocks(inode, nrblocks);
Mingming Caoee12b632008-08-19 22:16:05 -04002480}
Alex Tomasa86c6182006-10-11 01:21:03 -07002481
Mingming Caoee12b632008-08-19 22:16:05 -04002482/*
Jan Karafffb2732013-06-04 13:01:11 -04002483 * How many index/leaf blocks need to change/allocate to add @extents extents?
Mingming Caoee12b632008-08-19 22:16:05 -04002484 *
Jan Karafffb2732013-06-04 13:01:11 -04002485 * If we add a single extent, then in the worse case, each tree level
2486 * index/leaf need to be changed in case of the tree split.
Mingming Caoee12b632008-08-19 22:16:05 -04002487 *
Jan Karafffb2732013-06-04 13:01:11 -04002488 * If more extents are inserted, they could cause the whole tree split more
2489 * than once, but this is really rare.
Mingming Caoee12b632008-08-19 22:16:05 -04002490 */
Jan Karafffb2732013-06-04 13:01:11 -04002491int ext4_ext_index_trans_blocks(struct inode *inode, int extents)
Mingming Caoee12b632008-08-19 22:16:05 -04002492{
2493 int index;
Tao Maf19d5872012-12-10 14:05:51 -05002494 int depth;
2495
2496 /* If we are converting the inline data, only one is needed here. */
2497 if (ext4_has_inline_data(inode))
2498 return 1;
2499
2500 depth = ext_depth(inode);
Alex Tomasa86c6182006-10-11 01:21:03 -07002501
Jan Karafffb2732013-06-04 13:01:11 -04002502 if (extents <= 1)
Mingming Caoee12b632008-08-19 22:16:05 -04002503 index = depth * 2;
2504 else
2505 index = depth * 3;
Alex Tomasa86c6182006-10-11 01:21:03 -07002506
Mingming Caoee12b632008-08-19 22:16:05 -04002507 return index;
Alex Tomasa86c6182006-10-11 01:21:03 -07002508}
2509
Theodore Ts'o981250c2013-06-12 11:48:29 -04002510static inline int get_default_free_blocks_flags(struct inode *inode)
2511{
Tahsin Erdoganddfa17e2017-06-21 21:36:51 -04002512 if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode) ||
2513 ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE))
Theodore Ts'o981250c2013-06-12 11:48:29 -04002514 return EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET;
2515 else if (ext4_should_journal_data(inode))
2516 return EXT4_FREE_BLOCKS_FORGET;
2517 return 0;
2518}
2519
Eric Whitney9fe67142018-10-01 14:25:08 -04002520/*
2521 * ext4_rereserve_cluster - increment the reserved cluster count when
2522 * freeing a cluster with a pending reservation
2523 *
2524 * @inode - file containing the cluster
2525 * @lblk - logical block in cluster to be reserved
2526 *
2527 * Increments the reserved cluster count and adjusts quota in a bigalloc
2528 * file system when freeing a partial cluster containing at least one
2529 * delayed and unwritten block. A partial cluster meeting that
2530 * requirement will have a pending reservation. If so, the
2531 * RERESERVE_CLUSTER flag is used when calling ext4_free_blocks() to
2532 * defer reserved and allocated space accounting to a subsequent call
2533 * to this function.
2534 */
2535static void ext4_rereserve_cluster(struct inode *inode, ext4_lblk_t lblk)
2536{
2537 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
2538 struct ext4_inode_info *ei = EXT4_I(inode);
2539
2540 dquot_reclaim_block(inode, EXT4_C2B(sbi, 1));
2541
2542 spin_lock(&ei->i_block_reservation_lock);
2543 ei->i_reserved_data_blocks++;
2544 percpu_counter_add(&sbi->s_dirtyclusters_counter, 1);
2545 spin_unlock(&ei->i_block_reservation_lock);
2546
2547 percpu_counter_add(&sbi->s_freeclusters_counter, 1);
2548 ext4_remove_pending(inode, lblk);
2549}
2550
Alex Tomasa86c6182006-10-11 01:21:03 -07002551static int ext4_remove_blocks(handle_t *handle, struct inode *inode,
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002552 struct ext4_extent *ex,
Eric Whitney9fe67142018-10-01 14:25:08 -04002553 struct partial_cluster *partial,
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002554 ext4_lblk_t from, ext4_lblk_t to)
Alex Tomasa86c6182006-10-11 01:21:03 -07002555{
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002556 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Eric Whitney345ee942014-11-23 00:59:39 -05002557 unsigned short ee_len = ext4_ext_get_actual_len(ex);
Eric Whitney9fe67142018-10-01 14:25:08 -04002558 ext4_fsblk_t last_pblk, pblk;
2559 ext4_lblk_t num;
2560 int flags;
2561
2562 /* only extent tail removal is allowed */
2563 if (from < le32_to_cpu(ex->ee_block) ||
2564 to != le32_to_cpu(ex->ee_block) + ee_len - 1) {
2565 ext4_error(sbi->s_sb,
2566 "strange request: removal(2) %u-%u from %u:%u",
2567 from, to, le32_to_cpu(ex->ee_block), ee_len);
2568 return 0;
2569 }
2570
2571#ifdef EXTENTS_STATS
2572 spin_lock(&sbi->s_ext_stats_lock);
2573 sbi->s_ext_blocks += ee_len;
2574 sbi->s_ext_extents++;
2575 if (ee_len < sbi->s_ext_min)
2576 sbi->s_ext_min = ee_len;
2577 if (ee_len > sbi->s_ext_max)
2578 sbi->s_ext_max = ee_len;
2579 if (ext_depth(inode) > sbi->s_depth_max)
2580 sbi->s_depth_max = ext_depth(inode);
2581 spin_unlock(&sbi->s_ext_stats_lock);
2582#endif
2583
2584 trace_ext4_remove_blocks(inode, ex, from, to, partial);
2585
2586 /*
2587 * if we have a partial cluster, and it's different from the
2588 * cluster of the last block in the extent, we free it
2589 */
2590 last_pblk = ext4_ext_pblock(ex) + ee_len - 1;
2591
2592 if (partial->state != initial &&
2593 partial->pclu != EXT4_B2C(sbi, last_pblk)) {
2594 if (partial->state == tofree) {
2595 flags = get_default_free_blocks_flags(inode);
2596 if (ext4_is_pending(inode, partial->lblk))
2597 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
2598 ext4_free_blocks(handle, inode, NULL,
2599 EXT4_C2B(sbi, partial->pclu),
2600 sbi->s_cluster_ratio, flags);
2601 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
2602 ext4_rereserve_cluster(inode, partial->lblk);
2603 }
2604 partial->state = initial;
2605 }
2606
2607 num = le32_to_cpu(ex->ee_block) + ee_len - from;
2608 pblk = ext4_ext_pblock(ex) + ee_len - num;
2609
2610 /*
2611 * We free the partial cluster at the end of the extent (if any),
2612 * unless the cluster is used by another extent (partial_cluster
2613 * state is nofree). If a partial cluster exists here, it must be
2614 * shared with the last block in the extent.
2615 */
2616 flags = get_default_free_blocks_flags(inode);
2617
2618 /* partial, left end cluster aligned, right end unaligned */
2619 if ((EXT4_LBLK_COFF(sbi, to) != sbi->s_cluster_ratio - 1) &&
2620 (EXT4_LBLK_CMASK(sbi, to) >= from) &&
2621 (partial->state != nofree)) {
2622 if (ext4_is_pending(inode, to))
2623 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
2624 ext4_free_blocks(handle, inode, NULL,
2625 EXT4_PBLK_CMASK(sbi, last_pblk),
2626 sbi->s_cluster_ratio, flags);
2627 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
2628 ext4_rereserve_cluster(inode, to);
2629 partial->state = initial;
2630 flags = get_default_free_blocks_flags(inode);
2631 }
2632
2633 flags |= EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER;
Andrey Sidorov18888cf2012-09-19 14:14:53 -04002634
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002635 /*
2636 * For bigalloc file systems, we never free a partial cluster
Eric Whitney9fe67142018-10-01 14:25:08 -04002637 * at the beginning of the extent. Instead, we check to see if we
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002638 * need to free it on a subsequent call to ext4_remove_blocks,
Eric Whitney345ee942014-11-23 00:59:39 -05002639 * or at the end of ext4_ext_rm_leaf or ext4_ext_remove_space.
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002640 */
2641 flags |= EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER;
Eric Whitney9fe67142018-10-01 14:25:08 -04002642 ext4_free_blocks(handle, inode, NULL, pblk, num, flags);
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002643
Eric Whitney9fe67142018-10-01 14:25:08 -04002644 /* reset the partial cluster if we've freed past it */
2645 if (partial->state != initial && partial->pclu != EXT4_B2C(sbi, pblk))
2646 partial->state = initial;
2647
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002648 /*
Eric Whitney9fe67142018-10-01 14:25:08 -04002649 * If we've freed the entire extent but the beginning is not left
2650 * cluster aligned and is not marked as ineligible for freeing we
2651 * record the partial cluster at the beginning of the extent. It
2652 * wasn't freed by the preceding ext4_free_blocks() call, and we
2653 * need to look farther to the left to determine if it's to be freed
2654 * (not shared with another extent). Else, reset the partial
2655 * cluster - we're either done freeing or the beginning of the
2656 * extent is left cluster aligned.
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002657 */
Eric Whitney9fe67142018-10-01 14:25:08 -04002658 if (EXT4_LBLK_COFF(sbi, from) && num == ee_len) {
2659 if (partial->state == initial) {
2660 partial->pclu = EXT4_B2C(sbi, pblk);
2661 partial->lblk = from;
2662 partial->state = tofree;
Eric Whitney345ee942014-11-23 00:59:39 -05002663 }
Eric Whitney9fe67142018-10-01 14:25:08 -04002664 } else {
2665 partial->state = initial;
2666 }
2667
Alex Tomasa86c6182006-10-11 01:21:03 -07002668 return 0;
2669}
2670
Allison Hendersond583fb82011-05-25 07:41:43 -04002671/*
2672 * ext4_ext_rm_leaf() Removes the extents associated with the
Eric Whitney5bf43762014-11-23 00:58:11 -05002673 * blocks appearing between "start" and "end". Both "start"
2674 * and "end" must appear in the same extent or EIO is returned.
Allison Hendersond583fb82011-05-25 07:41:43 -04002675 *
2676 * @handle: The journal handle
2677 * @inode: The files inode
2678 * @path: The path to the leaf
Lukas Czernerd23142c2013-05-27 23:33:35 -04002679 * @partial_cluster: The cluster which we'll have to free if all extents
Eric Whitney5bf43762014-11-23 00:58:11 -05002680 * has been released from it. However, if this value is
2681 * negative, it's a cluster just to the right of the
2682 * punched region and it must not be freed.
Allison Hendersond583fb82011-05-25 07:41:43 -04002683 * @start: The first block to remove
2684 * @end: The last block to remove
2685 */
Alex Tomasa86c6182006-10-11 01:21:03 -07002686static int
2687ext4_ext_rm_leaf(handle_t *handle, struct inode *inode,
Lukas Czernerd23142c2013-05-27 23:33:35 -04002688 struct ext4_ext_path *path,
Eric Whitney9fe67142018-10-01 14:25:08 -04002689 struct partial_cluster *partial,
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002690 ext4_lblk_t start, ext4_lblk_t end)
Alex Tomasa86c6182006-10-11 01:21:03 -07002691{
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002692 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Alex Tomasa86c6182006-10-11 01:21:03 -07002693 int err = 0, correct_index = 0;
Jan Kara83448bd2019-11-05 17:44:29 +01002694 int depth = ext_depth(inode), credits, revoke_credits;
Alex Tomasa86c6182006-10-11 01:21:03 -07002695 struct ext4_extent_header *eh;
Dmitry Monakhov750c9c42011-10-25 05:35:05 -04002696 ext4_lblk_t a, b;
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -05002697 unsigned num;
2698 ext4_lblk_t ex_ee_block;
Alex Tomasa86c6182006-10-11 01:21:03 -07002699 unsigned short ex_ee_len;
Lukas Czerner556615d2014-04-20 23:45:47 -04002700 unsigned unwritten = 0;
Alex Tomasa86c6182006-10-11 01:21:03 -07002701 struct ext4_extent *ex;
Lukas Czernerd23142c2013-05-27 23:33:35 -04002702 ext4_fsblk_t pblk;
Alex Tomasa86c6182006-10-11 01:21:03 -07002703
Alex Tomasc29c0ae2007-07-18 09:19:09 -04002704 /* the header must be checked already in ext4_ext_remove_space() */
Lukas Czerner5f95d212012-03-19 23:03:19 -04002705 ext_debug("truncate since %u in leaf to %u\n", start, end);
Alex Tomasa86c6182006-10-11 01:21:03 -07002706 if (!path[depth].p_hdr)
2707 path[depth].p_hdr = ext_block_hdr(path[depth].p_bh);
2708 eh = path[depth].p_hdr;
Frank Mayhar273df552010-03-02 11:46:09 -05002709 if (unlikely(path[depth].p_hdr == NULL)) {
2710 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04002711 return -EFSCORRUPTED;
Frank Mayhar273df552010-03-02 11:46:09 -05002712 }
Alex Tomasa86c6182006-10-11 01:21:03 -07002713 /* find where to start removing */
Ashish Sangwan6ae06ff2013-07-01 08:12:41 -04002714 ex = path[depth].p_ext;
2715 if (!ex)
2716 ex = EXT_LAST_EXTENT(eh);
Alex Tomasa86c6182006-10-11 01:21:03 -07002717
2718 ex_ee_block = le32_to_cpu(ex->ee_block);
Amit Aroraa2df2a62007-07-17 21:42:41 -04002719 ex_ee_len = ext4_ext_get_actual_len(ex);
Alex Tomasa86c6182006-10-11 01:21:03 -07002720
Eric Whitney9fe67142018-10-01 14:25:08 -04002721 trace_ext4_ext_rm_leaf(inode, start, ex, partial);
Aditya Kalid8990242011-09-09 19:18:51 -04002722
Alex Tomasa86c6182006-10-11 01:21:03 -07002723 while (ex >= EXT_FIRST_EXTENT(eh) &&
2724 ex_ee_block + ex_ee_len > start) {
Aneesh Kumar K.Va41f2072009-06-10 14:22:55 -04002725
Lukas Czerner556615d2014-04-20 23:45:47 -04002726 if (ext4_ext_is_unwritten(ex))
2727 unwritten = 1;
Aneesh Kumar K.Va41f2072009-06-10 14:22:55 -04002728 else
Lukas Czerner556615d2014-04-20 23:45:47 -04002729 unwritten = 0;
Aneesh Kumar K.Va41f2072009-06-10 14:22:55 -04002730
Mingming553f9002009-09-18 13:34:55 -04002731 ext_debug("remove ext %u:[%d]%d\n", ex_ee_block,
Lukas Czerner556615d2014-04-20 23:45:47 -04002732 unwritten, ex_ee_len);
Alex Tomasa86c6182006-10-11 01:21:03 -07002733 path[depth].p_ext = ex;
2734
2735 a = ex_ee_block > start ? ex_ee_block : start;
Allison Hendersond583fb82011-05-25 07:41:43 -04002736 b = ex_ee_block+ex_ee_len - 1 < end ?
2737 ex_ee_block+ex_ee_len - 1 : end;
Alex Tomasa86c6182006-10-11 01:21:03 -07002738
2739 ext_debug(" border %u:%u\n", a, b);
2740
Allison Hendersond583fb82011-05-25 07:41:43 -04002741 /* If this extent is beyond the end of the hole, skip it */
Lukas Czerner5f95d212012-03-19 23:03:19 -04002742 if (end < ex_ee_block) {
Lukas Czernerd23142c2013-05-27 23:33:35 -04002743 /*
2744 * We're going to skip this extent and move to another,
Eric Whitneyf4226d92014-11-23 00:55:42 -05002745 * so note that its first cluster is in use to avoid
2746 * freeing it when removing blocks. Eventually, the
2747 * right edge of the truncated/punched region will
2748 * be just to the left.
Lukas Czernerd23142c2013-05-27 23:33:35 -04002749 */
Eric Whitneyf4226d92014-11-23 00:55:42 -05002750 if (sbi->s_cluster_ratio > 1) {
2751 pblk = ext4_ext_pblock(ex);
Eric Whitney9fe67142018-10-01 14:25:08 -04002752 partial->pclu = EXT4_B2C(sbi, pblk);
2753 partial->state = nofree;
Eric Whitneyf4226d92014-11-23 00:55:42 -05002754 }
Allison Hendersond583fb82011-05-25 07:41:43 -04002755 ex--;
2756 ex_ee_block = le32_to_cpu(ex->ee_block);
2757 ex_ee_len = ext4_ext_get_actual_len(ex);
2758 continue;
Dmitry Monakhov750c9c42011-10-25 05:35:05 -04002759 } else if (b != ex_ee_block + ex_ee_len - 1) {
Lukas Czernerdc1841d2012-03-19 23:07:43 -04002760 EXT4_ERROR_INODE(inode,
2761 "can not handle truncate %u:%u "
2762 "on extent %u:%u",
2763 start, end, ex_ee_block,
2764 ex_ee_block + ex_ee_len - 1);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04002765 err = -EFSCORRUPTED;
Dmitry Monakhov750c9c42011-10-25 05:35:05 -04002766 goto out;
Alex Tomasa86c6182006-10-11 01:21:03 -07002767 } else if (a != ex_ee_block) {
2768 /* remove tail of the extent */
Dmitry Monakhov750c9c42011-10-25 05:35:05 -04002769 num = a - ex_ee_block;
Alex Tomasa86c6182006-10-11 01:21:03 -07002770 } else {
2771 /* remove whole extent: excellent! */
Alex Tomasa86c6182006-10-11 01:21:03 -07002772 num = 0;
Alex Tomasa86c6182006-10-11 01:21:03 -07002773 }
Theodore Ts'o34071da2008-08-01 21:59:19 -04002774 /*
2775 * 3 for leaf, sb, and inode plus 2 (bmap and group
2776 * descriptor) for each block group; assume two block
2777 * groups plus ex_ee_len/blocks_per_block_group for
2778 * the worst case
2779 */
2780 credits = 7 + 2*(ex_ee_len/EXT4_BLOCKS_PER_GROUP(inode->i_sb));
Alex Tomasa86c6182006-10-11 01:21:03 -07002781 if (ex == EXT_FIRST_EXTENT(eh)) {
2782 correct_index = 1;
2783 credits += (ext_depth(inode)) + 1;
2784 }
Dmitry Monakhov5aca07e2009-12-08 22:42:15 -05002785 credits += EXT4_MAXQUOTAS_TRANS_BLOCKS(inode->i_sb);
Jan Kara83448bd2019-11-05 17:44:29 +01002786 /*
2787 * We may end up freeing some index blocks and data from the
2788 * punched range. Note that partial clusters are accounted for
2789 * by ext4_free_data_revoke_credits().
2790 */
2791 revoke_credits =
2792 ext4_free_metadata_revoke_credits(inode->i_sb,
2793 ext_depth(inode)) +
2794 ext4_free_data_revoke_credits(inode, b - a + 1);
Alex Tomasa86c6182006-10-11 01:21:03 -07002795
Jan Karaa4130362019-11-05 17:44:16 +01002796 err = ext4_datasem_ensure_credits(handle, inode, credits,
Jan Kara83448bd2019-11-05 17:44:29 +01002797 credits, revoke_credits);
Jan Karaa4130362019-11-05 17:44:16 +01002798 if (err) {
2799 if (err > 0)
2800 err = -EAGAIN;
Alex Tomasa86c6182006-10-11 01:21:03 -07002801 goto out;
Jan Karaa4130362019-11-05 17:44:16 +01002802 }
Alex Tomasa86c6182006-10-11 01:21:03 -07002803
2804 err = ext4_ext_get_access(handle, inode, path + depth);
2805 if (err)
2806 goto out;
2807
Eric Whitney9fe67142018-10-01 14:25:08 -04002808 err = ext4_remove_blocks(handle, inode, ex, partial, a, b);
Alex Tomasa86c6182006-10-11 01:21:03 -07002809 if (err)
2810 goto out;
2811
Dmitry Monakhov750c9c42011-10-25 05:35:05 -04002812 if (num == 0)
Randy Dunlapd0d856e2006-10-11 01:21:07 -07002813 /* this extent is removed; mark slot entirely unused */
Alex Tomasf65e6fb2006-10-11 01:21:05 -07002814 ext4_ext_store_pblock(ex, 0);
Alex Tomasa86c6182006-10-11 01:21:03 -07002815
Alex Tomasa86c6182006-10-11 01:21:03 -07002816 ex->ee_len = cpu_to_le16(num);
Amit Arora749269f2007-07-18 09:02:56 -04002817 /*
Lukas Czerner556615d2014-04-20 23:45:47 -04002818 * Do not mark unwritten if all the blocks in the
Amit Arora749269f2007-07-18 09:02:56 -04002819 * extent have been removed.
2820 */
Lukas Czerner556615d2014-04-20 23:45:47 -04002821 if (unwritten && num)
2822 ext4_ext_mark_unwritten(ex);
Allison Hendersond583fb82011-05-25 07:41:43 -04002823 /*
2824 * If the extent was completely released,
2825 * we need to remove it from the leaf
2826 */
2827 if (num == 0) {
Lukas Czernerf17722f2011-06-06 00:05:17 -04002828 if (end != EXT_MAX_BLOCKS - 1) {
Allison Hendersond583fb82011-05-25 07:41:43 -04002829 /*
2830 * For hole punching, we need to scoot all the
2831 * extents up when an extent is removed so that
2832 * we dont have blank extents in the middle
2833 */
2834 memmove(ex, ex+1, (EXT_LAST_EXTENT(eh) - ex) *
2835 sizeof(struct ext4_extent));
2836
2837 /* Now get rid of the one at the end */
2838 memset(EXT_LAST_EXTENT(eh), 0,
2839 sizeof(struct ext4_extent));
2840 }
2841 le16_add_cpu(&eh->eh_entries, -1);
Eric Whitney5bf43762014-11-23 00:58:11 -05002842 }
Allison Hendersond583fb82011-05-25 07:41:43 -04002843
Dmitry Monakhov750c9c42011-10-25 05:35:05 -04002844 err = ext4_ext_dirty(handle, inode, path + depth);
2845 if (err)
2846 goto out;
2847
Yongqiang Yangbf52c6f2011-11-01 18:59:26 -04002848 ext_debug("new extent: %u:%u:%llu\n", ex_ee_block, num,
Theodore Ts'obf89d162010-10-27 21:30:14 -04002849 ext4_ext_pblock(ex));
Alex Tomasa86c6182006-10-11 01:21:03 -07002850 ex--;
2851 ex_ee_block = le32_to_cpu(ex->ee_block);
Amit Aroraa2df2a62007-07-17 21:42:41 -04002852 ex_ee_len = ext4_ext_get_actual_len(ex);
Alex Tomasa86c6182006-10-11 01:21:03 -07002853 }
2854
2855 if (correct_index && eh->eh_entries)
2856 err = ext4_ext_correct_indexes(handle, inode, path);
2857
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002858 /*
Eric Whitneyad6599a2014-04-01 19:49:30 -04002859 * If there's a partial cluster and at least one extent remains in
2860 * the leaf, free the partial cluster if it isn't shared with the
Eric Whitney5bf43762014-11-23 00:58:11 -05002861 * current extent. If it is shared with the current extent
Eric Whitney9fe67142018-10-01 14:25:08 -04002862 * we reset the partial cluster because we've reached the start of the
Eric Whitney5bf43762014-11-23 00:58:11 -05002863 * truncated/punched region and we're done removing blocks.
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002864 */
Eric Whitney9fe67142018-10-01 14:25:08 -04002865 if (partial->state == tofree && ex >= EXT_FIRST_EXTENT(eh)) {
Eric Whitney5bf43762014-11-23 00:58:11 -05002866 pblk = ext4_ext_pblock(ex) + ex_ee_len - 1;
Eric Whitney9fe67142018-10-01 14:25:08 -04002867 if (partial->pclu != EXT4_B2C(sbi, pblk)) {
2868 int flags = get_default_free_blocks_flags(inode);
2869
2870 if (ext4_is_pending(inode, partial->lblk))
2871 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
Eric Whitney5bf43762014-11-23 00:58:11 -05002872 ext4_free_blocks(handle, inode, NULL,
Eric Whitney9fe67142018-10-01 14:25:08 -04002873 EXT4_C2B(sbi, partial->pclu),
2874 sbi->s_cluster_ratio, flags);
2875 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
2876 ext4_rereserve_cluster(inode, partial->lblk);
Eric Whitney5bf43762014-11-23 00:58:11 -05002877 }
Eric Whitney9fe67142018-10-01 14:25:08 -04002878 partial->state = initial;
Theodore Ts'o0aa06002011-09-09 18:54:51 -04002879 }
2880
Alex Tomasa86c6182006-10-11 01:21:03 -07002881 /* if this leaf is free, then we should
2882 * remove it from index block above */
2883 if (err == 0 && eh->eh_entries == 0 && path[depth].p_bh != NULL)
Forrest Liuc36575e2012-12-17 09:55:39 -05002884 err = ext4_ext_rm_idx(handle, inode, path, depth);
Alex Tomasa86c6182006-10-11 01:21:03 -07002885
2886out:
2887 return err;
2888}
2889
2890/*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07002891 * ext4_ext_more_to_rm:
2892 * returns 1 if current index has to be freed (even partial)
Alex Tomasa86c6182006-10-11 01:21:03 -07002893 */
Avantika Mathur09b88252006-12-06 20:41:36 -08002894static int
Alex Tomasa86c6182006-10-11 01:21:03 -07002895ext4_ext_more_to_rm(struct ext4_ext_path *path)
2896{
2897 BUG_ON(path->p_idx == NULL);
2898
2899 if (path->p_idx < EXT_FIRST_INDEX(path->p_hdr))
2900 return 0;
2901
2902 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07002903 * if truncate on deeper level happened, it wasn't partial,
Alex Tomasa86c6182006-10-11 01:21:03 -07002904 * so we have to consider current index for truncation
2905 */
2906 if (le16_to_cpu(path->p_hdr->eh_entries) == path->p_block)
2907 return 0;
2908 return 1;
2909}
2910
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04002911int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start,
2912 ext4_lblk_t end)
Alex Tomasa86c6182006-10-11 01:21:03 -07002913{
Eric Whitneyf4226d92014-11-23 00:55:42 -05002914 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Alex Tomasa86c6182006-10-11 01:21:03 -07002915 int depth = ext_depth(inode);
Ashish Sangwan968dee72012-07-22 22:49:08 -04002916 struct ext4_ext_path *path = NULL;
Eric Whitney9fe67142018-10-01 14:25:08 -04002917 struct partial_cluster partial;
Alex Tomasa86c6182006-10-11 01:21:03 -07002918 handle_t *handle;
Dmitry Monakhov6f2080e2012-09-30 23:03:50 -04002919 int i = 0, err = 0;
Alex Tomasa86c6182006-10-11 01:21:03 -07002920
Eric Whitney9fe67142018-10-01 14:25:08 -04002921 partial.pclu = 0;
2922 partial.lblk = 0;
2923 partial.state = initial;
2924
Lukas Czerner5f95d212012-03-19 23:03:19 -04002925 ext_debug("truncate since %u to %u\n", start, end);
Alex Tomasa86c6182006-10-11 01:21:03 -07002926
2927 /* probably first extent we're gonna free will be last in block */
Jan Kara83448bd2019-11-05 17:44:29 +01002928 handle = ext4_journal_start_with_revoke(inode, EXT4_HT_TRUNCATE,
2929 depth + 1,
2930 ext4_free_metadata_revoke_credits(inode->i_sb, depth));
Alex Tomasa86c6182006-10-11 01:21:03 -07002931 if (IS_ERR(handle))
2932 return PTR_ERR(handle);
2933
Dmitry Monakhov0617b832010-05-17 01:00:00 -04002934again:
Lukas Czerner61801322013-05-27 23:32:35 -04002935 trace_ext4_ext_remove_space(inode, start, end, depth);
Aditya Kalid8990242011-09-09 19:18:51 -04002936
Alex Tomasa86c6182006-10-11 01:21:03 -07002937 /*
Lukas Czerner5f95d212012-03-19 23:03:19 -04002938 * Check if we are removing extents inside the extent tree. If that
2939 * is the case, we are going to punch a hole inside the extent tree
2940 * so we have to check whether we need to split the extent covering
2941 * the last block to remove so we can easily remove the part of it
2942 * in ext4_ext_rm_leaf().
2943 */
2944 if (end < EXT_MAX_BLOCKS - 1) {
2945 struct ext4_extent *ex;
Eric Whitneyf4226d92014-11-23 00:55:42 -05002946 ext4_lblk_t ee_block, ex_end, lblk;
2947 ext4_fsblk_t pblk;
Lukas Czerner5f95d212012-03-19 23:03:19 -04002948
Eric Whitneyf4226d92014-11-23 00:55:42 -05002949 /* find extent for or closest extent to this block */
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04002950 path = ext4_find_extent(inode, end, NULL, EXT4_EX_NOCACHE);
Lukas Czerner5f95d212012-03-19 23:03:19 -04002951 if (IS_ERR(path)) {
2952 ext4_journal_stop(handle);
2953 return PTR_ERR(path);
2954 }
2955 depth = ext_depth(inode);
Dmitry Monakhov6f2080e2012-09-30 23:03:50 -04002956 /* Leaf not may not exist only if inode has no blocks at all */
Lukas Czerner5f95d212012-03-19 23:03:19 -04002957 ex = path[depth].p_ext;
Ashish Sangwan968dee72012-07-22 22:49:08 -04002958 if (!ex) {
Dmitry Monakhov6f2080e2012-09-30 23:03:50 -04002959 if (depth) {
2960 EXT4_ERROR_INODE(inode,
2961 "path[%d].p_hdr == NULL",
2962 depth);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04002963 err = -EFSCORRUPTED;
Dmitry Monakhov6f2080e2012-09-30 23:03:50 -04002964 }
2965 goto out;
Ashish Sangwan968dee72012-07-22 22:49:08 -04002966 }
Lukas Czerner5f95d212012-03-19 23:03:19 -04002967
2968 ee_block = le32_to_cpu(ex->ee_block);
Eric Whitneyf4226d92014-11-23 00:55:42 -05002969 ex_end = ee_block + ext4_ext_get_actual_len(ex) - 1;
Lukas Czerner5f95d212012-03-19 23:03:19 -04002970
2971 /*
2972 * See if the last block is inside the extent, if so split
2973 * the extent at 'end' block so we can easily remove the
2974 * tail of the first part of the split extent in
2975 * ext4_ext_rm_leaf().
2976 */
Eric Whitneyf4226d92014-11-23 00:55:42 -05002977 if (end >= ee_block && end < ex_end) {
2978
2979 /*
2980 * If we're going to split the extent, note that
2981 * the cluster containing the block after 'end' is
2982 * in use to avoid freeing it when removing blocks.
2983 */
2984 if (sbi->s_cluster_ratio > 1) {
2985 pblk = ext4_ext_pblock(ex) + end - ee_block + 2;
Eric Whitney9fe67142018-10-01 14:25:08 -04002986 partial.pclu = EXT4_B2C(sbi, pblk);
2987 partial.state = nofree;
Eric Whitneyf4226d92014-11-23 00:55:42 -05002988 }
2989
Lukas Czerner5f95d212012-03-19 23:03:19 -04002990 /*
2991 * Split the extent in two so that 'end' is the last
Lukas Czerner27dd4382013-04-09 22:11:22 -04002992 * block in the first new extent. Also we should not
2993 * fail removing space due to ENOSPC so try to use
2994 * reserved block if that happens.
Lukas Czerner5f95d212012-03-19 23:03:19 -04002995 */
Theodore Ts'odfe50802014-09-01 14:37:09 -04002996 err = ext4_force_split_extent_at(handle, inode, &path,
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04002997 end + 1, 1);
Lukas Czerner5f95d212012-03-19 23:03:19 -04002998 if (err < 0)
2999 goto out;
Eric Whitneyf4226d92014-11-23 00:55:42 -05003000
Eric Whitney7bd75232019-02-28 23:34:11 -05003001 } else if (sbi->s_cluster_ratio > 1 && end >= ex_end &&
3002 partial.state == initial) {
Eric Whitneyf4226d92014-11-23 00:55:42 -05003003 /*
Eric Whitney7bd75232019-02-28 23:34:11 -05003004 * If we're punching, there's an extent to the right.
3005 * If the partial cluster hasn't been set, set it to
3006 * that extent's first cluster and its state to nofree
3007 * so it won't be freed should it contain blocks to be
3008 * removed. If it's already set (tofree/nofree), we're
3009 * retrying and keep the original partial cluster info
3010 * so a cluster marked tofree as a result of earlier
3011 * extent removal is not lost.
Eric Whitneyf4226d92014-11-23 00:55:42 -05003012 */
3013 lblk = ex_end + 1;
3014 err = ext4_ext_search_right(inode, path, &lblk, &pblk,
3015 &ex);
3016 if (err)
3017 goto out;
Eric Whitney9fe67142018-10-01 14:25:08 -04003018 if (pblk) {
3019 partial.pclu = EXT4_B2C(sbi, pblk);
3020 partial.state = nofree;
3021 }
Lukas Czerner5f95d212012-03-19 23:03:19 -04003022 }
Lukas Czerner5f95d212012-03-19 23:03:19 -04003023 }
Lukas Czerner5f95d212012-03-19 23:03:19 -04003024 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07003025 * We start scanning from right side, freeing all the blocks
3026 * after i_size and walking into the tree depth-wise.
Alex Tomasa86c6182006-10-11 01:21:03 -07003027 */
Dmitry Monakhov0617b832010-05-17 01:00:00 -04003028 depth = ext_depth(inode);
Ashish Sangwan968dee72012-07-22 22:49:08 -04003029 if (path) {
3030 int k = i = depth;
3031 while (--k > 0)
3032 path[k].p_block =
3033 le16_to_cpu(path[k].p_hdr->eh_entries)+1;
3034 } else {
Kees Cook6396bb22018-06-12 14:03:40 -07003035 path = kcalloc(depth + 1, sizeof(struct ext4_ext_path),
Ashish Sangwan968dee72012-07-22 22:49:08 -04003036 GFP_NOFS);
3037 if (path == NULL) {
3038 ext4_journal_stop(handle);
3039 return -ENOMEM;
3040 }
Theodore Ts'o10809df82014-09-01 14:40:09 -04003041 path[0].p_maxdepth = path[0].p_depth = depth;
Ashish Sangwan968dee72012-07-22 22:49:08 -04003042 path[0].p_hdr = ext_inode_hdr(inode);
Theodore Ts'o89a4e482012-08-17 08:54:52 -04003043 i = 0;
Lukas Czerner5f95d212012-03-19 23:03:19 -04003044
Theodore Ts'oc3491792013-08-16 21:21:41 -04003045 if (ext4_ext_check(inode, path[0].p_hdr, depth, 0)) {
Darrick J. Wong6a797d22015-10-17 16:16:04 -04003046 err = -EFSCORRUPTED;
Ashish Sangwan968dee72012-07-22 22:49:08 -04003047 goto out;
3048 }
Alex Tomasa86c6182006-10-11 01:21:03 -07003049 }
Ashish Sangwan968dee72012-07-22 22:49:08 -04003050 err = 0;
Alex Tomasa86c6182006-10-11 01:21:03 -07003051
3052 while (i >= 0 && err == 0) {
3053 if (i == depth) {
3054 /* this is leaf block */
Allison Hendersond583fb82011-05-25 07:41:43 -04003055 err = ext4_ext_rm_leaf(handle, inode, path,
Eric Whitney9fe67142018-10-01 14:25:08 -04003056 &partial, start, end);
Randy Dunlapd0d856e2006-10-11 01:21:07 -07003057 /* root level has p_bh == NULL, brelse() eats this */
Alex Tomasa86c6182006-10-11 01:21:03 -07003058 brelse(path[i].p_bh);
3059 path[i].p_bh = NULL;
3060 i--;
3061 continue;
3062 }
3063
3064 /* this is index block */
3065 if (!path[i].p_hdr) {
3066 ext_debug("initialize header\n");
3067 path[i].p_hdr = ext_block_hdr(path[i].p_bh);
Alex Tomasa86c6182006-10-11 01:21:03 -07003068 }
3069
Alex Tomasa86c6182006-10-11 01:21:03 -07003070 if (!path[i].p_idx) {
Randy Dunlapd0d856e2006-10-11 01:21:07 -07003071 /* this level hasn't been touched yet */
Alex Tomasa86c6182006-10-11 01:21:03 -07003072 path[i].p_idx = EXT_LAST_INDEX(path[i].p_hdr);
3073 path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries)+1;
3074 ext_debug("init index ptr: hdr 0x%p, num %d\n",
3075 path[i].p_hdr,
3076 le16_to_cpu(path[i].p_hdr->eh_entries));
3077 } else {
Randy Dunlapd0d856e2006-10-11 01:21:07 -07003078 /* we were already here, see at next index */
Alex Tomasa86c6182006-10-11 01:21:03 -07003079 path[i].p_idx--;
3080 }
3081
3082 ext_debug("level %d - index, first 0x%p, cur 0x%p\n",
3083 i, EXT_FIRST_INDEX(path[i].p_hdr),
3084 path[i].p_idx);
3085 if (ext4_ext_more_to_rm(path + i)) {
Alex Tomasc29c0ae2007-07-18 09:19:09 -04003086 struct buffer_head *bh;
Alex Tomasa86c6182006-10-11 01:21:03 -07003087 /* go to the next level */
Mingming Cao2ae02102006-10-11 01:21:11 -07003088 ext_debug("move to level %d (block %llu)\n",
Theodore Ts'obf89d162010-10-27 21:30:14 -04003089 i + 1, ext4_idx_pblock(path[i].p_idx));
Alex Tomasa86c6182006-10-11 01:21:03 -07003090 memset(path + i + 1, 0, sizeof(*path));
Theodore Ts'o7d7ea892013-08-16 21:20:41 -04003091 bh = read_extent_tree_block(inode,
Theodore Ts'o107a7bd2013-08-16 21:23:41 -04003092 ext4_idx_pblock(path[i].p_idx), depth - i - 1,
3093 EXT4_EX_NOCACHE);
Theodore Ts'o7d7ea892013-08-16 21:20:41 -04003094 if (IS_ERR(bh)) {
Alex Tomasa86c6182006-10-11 01:21:03 -07003095 /* should we reset i_size? */
Theodore Ts'o7d7ea892013-08-16 21:20:41 -04003096 err = PTR_ERR(bh);
Alex Tomasa86c6182006-10-11 01:21:03 -07003097 break;
3098 }
Theodore Ts'o76828c882013-07-15 12:27:47 -04003099 /* Yield here to deal with large extent trees.
3100 * Should be a no-op if we did IO above. */
3101 cond_resched();
Alex Tomasc29c0ae2007-07-18 09:19:09 -04003102 if (WARN_ON(i + 1 > depth)) {
Darrick J. Wong6a797d22015-10-17 16:16:04 -04003103 err = -EFSCORRUPTED;
Alex Tomasc29c0ae2007-07-18 09:19:09 -04003104 break;
3105 }
Alex Tomasc29c0ae2007-07-18 09:19:09 -04003106 path[i + 1].p_bh = bh;
Alex Tomasa86c6182006-10-11 01:21:03 -07003107
Randy Dunlapd0d856e2006-10-11 01:21:07 -07003108 /* save actual number of indexes since this
3109 * number is changed at the next iteration */
Alex Tomasa86c6182006-10-11 01:21:03 -07003110 path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries);
3111 i++;
3112 } else {
Randy Dunlapd0d856e2006-10-11 01:21:07 -07003113 /* we finished processing this index, go up */
Alex Tomasa86c6182006-10-11 01:21:03 -07003114 if (path[i].p_hdr->eh_entries == 0 && i > 0) {
Randy Dunlapd0d856e2006-10-11 01:21:07 -07003115 /* index is empty, remove it;
Alex Tomasa86c6182006-10-11 01:21:03 -07003116 * handle must be already prepared by the
3117 * truncatei_leaf() */
Forrest Liuc36575e2012-12-17 09:55:39 -05003118 err = ext4_ext_rm_idx(handle, inode, path, i);
Alex Tomasa86c6182006-10-11 01:21:03 -07003119 }
Randy Dunlapd0d856e2006-10-11 01:21:07 -07003120 /* root level has p_bh == NULL, brelse() eats this */
Alex Tomasa86c6182006-10-11 01:21:03 -07003121 brelse(path[i].p_bh);
3122 path[i].p_bh = NULL;
3123 i--;
3124 ext_debug("return to level %d\n", i);
3125 }
3126 }
3127
Eric Whitney9fe67142018-10-01 14:25:08 -04003128 trace_ext4_ext_remove_space_done(inode, start, end, depth, &partial,
3129 path->p_hdr->eh_entries);
Aditya Kalid8990242011-09-09 19:18:51 -04003130
Eric Whitney0756b902014-11-23 00:59:39 -05003131 /*
Eric Whitney9fe67142018-10-01 14:25:08 -04003132 * if there's a partial cluster and we have removed the first extent
3133 * in the file, then we also free the partial cluster, if any
Eric Whitney0756b902014-11-23 00:59:39 -05003134 */
Eric Whitney9fe67142018-10-01 14:25:08 -04003135 if (partial.state == tofree && err == 0) {
3136 int flags = get_default_free_blocks_flags(inode);
3137
3138 if (ext4_is_pending(inode, partial.lblk))
3139 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
Aditya Kali7b415bf2011-09-09 19:04:51 -04003140 ext4_free_blocks(handle, inode, NULL,
Eric Whitney9fe67142018-10-01 14:25:08 -04003141 EXT4_C2B(sbi, partial.pclu),
3142 sbi->s_cluster_ratio, flags);
3143 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
3144 ext4_rereserve_cluster(inode, partial.lblk);
3145 partial.state = initial;
Aditya Kali7b415bf2011-09-09 19:04:51 -04003146 }
3147
Alex Tomasa86c6182006-10-11 01:21:03 -07003148 /* TODO: flexible tree reduction should be here */
3149 if (path->p_hdr->eh_entries == 0) {
3150 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07003151 * truncate to zero freed all the tree,
3152 * so we need to correct eh_depth
Alex Tomasa86c6182006-10-11 01:21:03 -07003153 */
3154 err = ext4_ext_get_access(handle, inode, path);
3155 if (err == 0) {
3156 ext_inode_hdr(inode)->eh_depth = 0;
3157 ext_inode_hdr(inode)->eh_max =
Theodore Ts'o55ad63b2009-08-28 10:40:33 -04003158 cpu_to_le16(ext4_ext_space_root(inode, 0));
Alex Tomasa86c6182006-10-11 01:21:03 -07003159 err = ext4_ext_dirty(handle, inode, path);
3160 }
3161 }
3162out:
Theodore Ts'ob7ea89a2014-09-01 14:39:09 -04003163 ext4_ext_drop_refs(path);
3164 kfree(path);
3165 path = NULL;
Theodore Ts'odfe50802014-09-01 14:37:09 -04003166 if (err == -EAGAIN)
3167 goto again;
Alex Tomasa86c6182006-10-11 01:21:03 -07003168 ext4_journal_stop(handle);
3169
3170 return err;
3171}
3172
3173/*
3174 * called at mount time
3175 */
3176void ext4_ext_init(struct super_block *sb)
3177{
3178 /*
3179 * possible initialization would be here
3180 */
3181
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04003182 if (ext4_has_feature_extents(sb)) {
Theodore Ts'o90576c02009-09-29 15:51:30 -04003183#if defined(AGGRESSIVE_TEST) || defined(CHECK_BINSEARCH) || defined(EXTENTS_STATS)
Theodore Ts'o92b97812012-03-19 23:41:49 -04003184 printk(KERN_INFO "EXT4-fs: file extents enabled"
Robert P. J. Daybbf2f9f2007-02-17 19:20:16 +01003185#ifdef AGGRESSIVE_TEST
Theodore Ts'o92b97812012-03-19 23:41:49 -04003186 ", aggressive tests"
Alex Tomasa86c6182006-10-11 01:21:03 -07003187#endif
3188#ifdef CHECK_BINSEARCH
Theodore Ts'o92b97812012-03-19 23:41:49 -04003189 ", check binsearch"
Alex Tomasa86c6182006-10-11 01:21:03 -07003190#endif
3191#ifdef EXTENTS_STATS
Theodore Ts'o92b97812012-03-19 23:41:49 -04003192 ", stats"
Alex Tomasa86c6182006-10-11 01:21:03 -07003193#endif
Theodore Ts'o92b97812012-03-19 23:41:49 -04003194 "\n");
Theodore Ts'o90576c02009-09-29 15:51:30 -04003195#endif
Alex Tomasa86c6182006-10-11 01:21:03 -07003196#ifdef EXTENTS_STATS
3197 spin_lock_init(&EXT4_SB(sb)->s_ext_stats_lock);
3198 EXT4_SB(sb)->s_ext_min = 1 << 30;
3199 EXT4_SB(sb)->s_ext_max = 0;
3200#endif
3201 }
3202}
3203
3204/*
3205 * called at umount time
3206 */
3207void ext4_ext_release(struct super_block *sb)
3208{
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04003209 if (!ext4_has_feature_extents(sb))
Alex Tomasa86c6182006-10-11 01:21:03 -07003210 return;
3211
3212#ifdef EXTENTS_STATS
3213 if (EXT4_SB(sb)->s_ext_blocks && EXT4_SB(sb)->s_ext_extents) {
3214 struct ext4_sb_info *sbi = EXT4_SB(sb);
3215 printk(KERN_ERR "EXT4-fs: %lu blocks in %lu extents (%lu ave)\n",
3216 sbi->s_ext_blocks, sbi->s_ext_extents,
3217 sbi->s_ext_blocks / sbi->s_ext_extents);
3218 printk(KERN_ERR "EXT4-fs: extents: %lu min, %lu max, max depth %lu\n",
3219 sbi->s_ext_min, sbi->s_ext_max, sbi->s_depth_max);
3220 }
3221#endif
3222}
3223
Zheng Liud7b2a002013-08-28 14:47:06 -04003224static int ext4_zeroout_es(struct inode *inode, struct ext4_extent *ex)
3225{
3226 ext4_lblk_t ee_block;
3227 ext4_fsblk_t ee_pblock;
3228 unsigned int ee_len;
3229
3230 ee_block = le32_to_cpu(ex->ee_block);
3231 ee_len = ext4_ext_get_actual_len(ex);
3232 ee_pblock = ext4_ext_pblock(ex);
3233
3234 if (ee_len == 0)
3235 return 0;
3236
3237 return ext4_es_insert_extent(inode, ee_block, ee_len, ee_pblock,
3238 EXTENT_STATUS_WRITTEN);
3239}
3240
Aneesh Kumar K.V093a0882008-04-29 08:11:12 -04003241/* FIXME!! we need to try to merge to left or right after zero-out */
3242static int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex)
3243{
Lukas Czerner24075182010-10-27 21:30:06 -04003244 ext4_fsblk_t ee_pblock;
3245 unsigned int ee_len;
Aneesh Kumar K.V093a0882008-04-29 08:11:12 -04003246
Aneesh Kumar K.V093a0882008-04-29 08:11:12 -04003247 ee_len = ext4_ext_get_actual_len(ex);
Theodore Ts'obf89d162010-10-27 21:30:14 -04003248 ee_pblock = ext4_ext_pblock(ex);
Jan Kara53085fa2015-12-07 15:09:35 -05003249 return ext4_issue_zeroout(inode, le32_to_cpu(ex->ee_block), ee_pblock,
3250 ee_len);
Aneesh Kumar K.V093a0882008-04-29 08:11:12 -04003251}
3252
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003253/*
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003254 * ext4_split_extent_at() splits an extent at given block.
3255 *
3256 * @handle: the journal handle
3257 * @inode: the file inode
3258 * @path: the path to the extent
3259 * @split: the logical block where the extent is splitted.
3260 * @split_flags: indicates if the extent could be zeroout if split fails, and
Lukas Czerner556615d2014-04-20 23:45:47 -04003261 * the states(init or unwritten) of new extents.
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003262 * @flags: flags used to insert new extent to extent tree.
3263 *
3264 *
3265 * Splits extent [a, b] into two extents [a, @split) and [@split, b], states
3266 * of which are deterimined by split_flag.
3267 *
3268 * There are two cases:
3269 * a> the extent are splitted into two extent.
3270 * b> split is not needed, and just mark the extent.
3271 *
3272 * return 0 on success.
3273 */
3274static int ext4_split_extent_at(handle_t *handle,
3275 struct inode *inode,
Theodore Ts'odfe50802014-09-01 14:37:09 -04003276 struct ext4_ext_path **ppath,
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003277 ext4_lblk_t split,
3278 int split_flag,
3279 int flags)
3280{
Theodore Ts'odfe50802014-09-01 14:37:09 -04003281 struct ext4_ext_path *path = *ppath;
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003282 ext4_fsblk_t newblock;
3283 ext4_lblk_t ee_block;
Zheng Liuadb23552013-03-10 21:13:05 -04003284 struct ext4_extent *ex, newex, orig_ex, zero_ex;
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003285 struct ext4_extent *ex2 = NULL;
3286 unsigned int ee_len, depth;
3287 int err = 0;
3288
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003289 BUG_ON((split_flag & (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)) ==
3290 (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2));
3291
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003292 ext_debug("ext4_split_extents_at: inode %lu, logical"
3293 "block %llu\n", inode->i_ino, (unsigned long long)split);
3294
3295 ext4_ext_show_leaf(inode, path);
3296
3297 depth = ext_depth(inode);
3298 ex = path[depth].p_ext;
3299 ee_block = le32_to_cpu(ex->ee_block);
3300 ee_len = ext4_ext_get_actual_len(ex);
3301 newblock = split - ee_block + ext4_ext_pblock(ex);
3302
3303 BUG_ON(split < ee_block || split >= (ee_block + ee_len));
Lukas Czerner556615d2014-04-20 23:45:47 -04003304 BUG_ON(!ext4_ext_is_unwritten(ex) &&
Dmitry Monakhov357b66f2013-03-04 00:34:34 -05003305 split_flag & (EXT4_EXT_MAY_ZEROOUT |
Lukas Czerner556615d2014-04-20 23:45:47 -04003306 EXT4_EXT_MARK_UNWRIT1 |
3307 EXT4_EXT_MARK_UNWRIT2));
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003308
3309 err = ext4_ext_get_access(handle, inode, path + depth);
3310 if (err)
3311 goto out;
3312
3313 if (split == ee_block) {
3314 /*
3315 * case b: block @split is the block that the extent begins with
3316 * then we just change the state of the extent, and splitting
3317 * is not needed.
3318 */
Lukas Czerner556615d2014-04-20 23:45:47 -04003319 if (split_flag & EXT4_EXT_MARK_UNWRIT2)
3320 ext4_ext_mark_unwritten(ex);
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003321 else
3322 ext4_ext_mark_initialized(ex);
3323
3324 if (!(flags & EXT4_GET_BLOCKS_PRE_IO))
Theodore Ts'oecb94f52012-08-17 09:44:17 -04003325 ext4_ext_try_to_merge(handle, inode, path, ex);
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003326
Theodore Ts'oecb94f52012-08-17 09:44:17 -04003327 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003328 goto out;
3329 }
3330
3331 /* case a */
3332 memcpy(&orig_ex, ex, sizeof(orig_ex));
3333 ex->ee_len = cpu_to_le16(split - ee_block);
Lukas Czerner556615d2014-04-20 23:45:47 -04003334 if (split_flag & EXT4_EXT_MARK_UNWRIT1)
3335 ext4_ext_mark_unwritten(ex);
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003336
3337 /*
3338 * path may lead to new leaf, not to original leaf any more
3339 * after ext4_ext_insert_extent() returns,
3340 */
3341 err = ext4_ext_dirty(handle, inode, path + depth);
3342 if (err)
3343 goto fix_extent_len;
3344
3345 ex2 = &newex;
3346 ex2->ee_block = cpu_to_le32(split);
3347 ex2->ee_len = cpu_to_le16(ee_len - (split - ee_block));
3348 ext4_ext_store_pblock(ex2, newblock);
Lukas Czerner556615d2014-04-20 23:45:47 -04003349 if (split_flag & EXT4_EXT_MARK_UNWRIT2)
3350 ext4_ext_mark_unwritten(ex2);
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003351
Theodore Ts'odfe50802014-09-01 14:37:09 -04003352 err = ext4_ext_insert_extent(handle, inode, ppath, &newex, flags);
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003353 if (err == -ENOSPC && (EXT4_EXT_MAY_ZEROOUT & split_flag)) {
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003354 if (split_flag & (EXT4_EXT_DATA_VALID1|EXT4_EXT_DATA_VALID2)) {
Zheng Liuadb23552013-03-10 21:13:05 -04003355 if (split_flag & EXT4_EXT_DATA_VALID1) {
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003356 err = ext4_ext_zeroout(inode, ex2);
Zheng Liuadb23552013-03-10 21:13:05 -04003357 zero_ex.ee_block = ex2->ee_block;
Zheng Liu8cde7ad2013-04-03 12:27:18 -04003358 zero_ex.ee_len = cpu_to_le16(
3359 ext4_ext_get_actual_len(ex2));
Zheng Liuadb23552013-03-10 21:13:05 -04003360 ext4_ext_store_pblock(&zero_ex,
3361 ext4_ext_pblock(ex2));
3362 } else {
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003363 err = ext4_ext_zeroout(inode, ex);
Zheng Liuadb23552013-03-10 21:13:05 -04003364 zero_ex.ee_block = ex->ee_block;
Zheng Liu8cde7ad2013-04-03 12:27:18 -04003365 zero_ex.ee_len = cpu_to_le16(
3366 ext4_ext_get_actual_len(ex));
Zheng Liuadb23552013-03-10 21:13:05 -04003367 ext4_ext_store_pblock(&zero_ex,
3368 ext4_ext_pblock(ex));
3369 }
3370 } else {
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003371 err = ext4_ext_zeroout(inode, &orig_ex);
Zheng Liuadb23552013-03-10 21:13:05 -04003372 zero_ex.ee_block = orig_ex.ee_block;
Zheng Liu8cde7ad2013-04-03 12:27:18 -04003373 zero_ex.ee_len = cpu_to_le16(
3374 ext4_ext_get_actual_len(&orig_ex));
Zheng Liuadb23552013-03-10 21:13:05 -04003375 ext4_ext_store_pblock(&zero_ex,
3376 ext4_ext_pblock(&orig_ex));
3377 }
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003378
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003379 if (err)
3380 goto fix_extent_len;
3381 /* update the extent length and mark as initialized */
Al Viroaf1584f2012-04-12 20:32:25 -04003382 ex->ee_len = cpu_to_le16(ee_len);
Theodore Ts'oecb94f52012-08-17 09:44:17 -04003383 ext4_ext_try_to_merge(handle, inode, path, ex);
3384 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
Zheng Liuadb23552013-03-10 21:13:05 -04003385 if (err)
3386 goto fix_extent_len;
3387
3388 /* update extent status tree */
Zheng Liud7b2a002013-08-28 14:47:06 -04003389 err = ext4_zeroout_es(inode, &zero_ex);
Zheng Liuadb23552013-03-10 21:13:05 -04003390
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003391 goto out;
3392 } else if (err)
3393 goto fix_extent_len;
3394
3395out:
3396 ext4_ext_show_leaf(inode, path);
3397 return err;
3398
3399fix_extent_len:
3400 ex->ee_len = orig_ex.ee_len;
Dmitry Monakhov29faed12014-07-27 22:30:29 -04003401 ext4_ext_dirty(handle, inode, path + path->p_depth);
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003402 return err;
3403}
3404
3405/*
3406 * ext4_split_extents() splits an extent and mark extent which is covered
3407 * by @map as split_flags indicates
3408 *
Anatol Pomozov70261f52013-08-28 14:40:12 -04003409 * It may result in splitting the extent into multiple extents (up to three)
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003410 * There are three possibilities:
3411 * a> There is no split required
3412 * b> Splits in two extents: Split is happening at either end of the extent
3413 * c> Splits in three extents: Somone is splitting in middle of the extent
3414 *
3415 */
3416static int ext4_split_extent(handle_t *handle,
3417 struct inode *inode,
Theodore Ts'odfe50802014-09-01 14:37:09 -04003418 struct ext4_ext_path **ppath,
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003419 struct ext4_map_blocks *map,
3420 int split_flag,
3421 int flags)
3422{
Theodore Ts'odfe50802014-09-01 14:37:09 -04003423 struct ext4_ext_path *path = *ppath;
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003424 ext4_lblk_t ee_block;
3425 struct ext4_extent *ex;
3426 unsigned int ee_len, depth;
3427 int err = 0;
Lukas Czerner556615d2014-04-20 23:45:47 -04003428 int unwritten;
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003429 int split_flag1, flags1;
Zheng Liu3a225672013-03-10 21:20:23 -04003430 int allocated = map->m_len;
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003431
3432 depth = ext_depth(inode);
3433 ex = path[depth].p_ext;
3434 ee_block = le32_to_cpu(ex->ee_block);
3435 ee_len = ext4_ext_get_actual_len(ex);
Lukas Czerner556615d2014-04-20 23:45:47 -04003436 unwritten = ext4_ext_is_unwritten(ex);
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003437
3438 if (map->m_lblk + map->m_len < ee_block + ee_len) {
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003439 split_flag1 = split_flag & EXT4_EXT_MAY_ZEROOUT;
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003440 flags1 = flags | EXT4_GET_BLOCKS_PRE_IO;
Lukas Czerner556615d2014-04-20 23:45:47 -04003441 if (unwritten)
3442 split_flag1 |= EXT4_EXT_MARK_UNWRIT1 |
3443 EXT4_EXT_MARK_UNWRIT2;
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003444 if (split_flag & EXT4_EXT_DATA_VALID2)
3445 split_flag1 |= EXT4_EXT_DATA_VALID1;
Theodore Ts'odfe50802014-09-01 14:37:09 -04003446 err = ext4_split_extent_at(handle, inode, ppath,
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003447 map->m_lblk + map->m_len, split_flag1, flags1);
Yongqiang Yang93917412011-05-22 20:49:12 -04003448 if (err)
3449 goto out;
Zheng Liu3a225672013-03-10 21:20:23 -04003450 } else {
3451 allocated = ee_len - (map->m_lblk - ee_block);
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003452 }
Dmitry Monakhov357b66f2013-03-04 00:34:34 -05003453 /*
3454 * Update path is required because previous ext4_split_extent_at() may
3455 * result in split of original leaf or extent zeroout.
3456 */
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04003457 path = ext4_find_extent(inode, map->m_lblk, ppath, 0);
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003458 if (IS_ERR(path))
3459 return PTR_ERR(path);
Dmitry Monakhov357b66f2013-03-04 00:34:34 -05003460 depth = ext_depth(inode);
3461 ex = path[depth].p_ext;
Dmitry Monakhova18ed352014-04-13 15:41:13 -04003462 if (!ex) {
3463 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
3464 (unsigned long) map->m_lblk);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04003465 return -EFSCORRUPTED;
Dmitry Monakhova18ed352014-04-13 15:41:13 -04003466 }
Lukas Czerner556615d2014-04-20 23:45:47 -04003467 unwritten = ext4_ext_is_unwritten(ex);
Dmitry Monakhov357b66f2013-03-04 00:34:34 -05003468 split_flag1 = 0;
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003469
3470 if (map->m_lblk >= ee_block) {
Dmitry Monakhov357b66f2013-03-04 00:34:34 -05003471 split_flag1 = split_flag & EXT4_EXT_DATA_VALID2;
Lukas Czerner556615d2014-04-20 23:45:47 -04003472 if (unwritten) {
3473 split_flag1 |= EXT4_EXT_MARK_UNWRIT1;
Dmitry Monakhov357b66f2013-03-04 00:34:34 -05003474 split_flag1 |= split_flag & (EXT4_EXT_MAY_ZEROOUT |
Lukas Czerner556615d2014-04-20 23:45:47 -04003475 EXT4_EXT_MARK_UNWRIT2);
Dmitry Monakhov357b66f2013-03-04 00:34:34 -05003476 }
Theodore Ts'odfe50802014-09-01 14:37:09 -04003477 err = ext4_split_extent_at(handle, inode, ppath,
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003478 map->m_lblk, split_flag1, flags);
3479 if (err)
3480 goto out;
3481 }
3482
3483 ext4_ext_show_leaf(inode, path);
3484out:
Zheng Liu3a225672013-03-10 21:20:23 -04003485 return err ? err : allocated;
Yongqiang Yang47ea3bb2011-05-03 12:23:07 -04003486}
3487
Amit Arora56055d32007-07-17 21:42:38 -04003488/*
Theodore Ts'oe35fd662010-05-16 19:00:00 -04003489 * This function is called by ext4_ext_map_blocks() if someone tries to write
Lukas Czerner556615d2014-04-20 23:45:47 -04003490 * to an unwritten extent. It may result in splitting the unwritten
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003491 * extent into multiple extents (up to three - one initialized and two
Lukas Czerner556615d2014-04-20 23:45:47 -04003492 * unwritten).
Amit Arora56055d32007-07-17 21:42:38 -04003493 * There are three possibilities:
3494 * a> There is no split required: Entire extent should be initialized
3495 * b> Splits in two extents: Write is happening at either end of the extent
3496 * c> Splits in three extents: Somone is writing in middle of the extent
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003497 *
3498 * Pre-conditions:
Lukas Czerner556615d2014-04-20 23:45:47 -04003499 * - The extent pointed to by 'path' is unwritten.
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003500 * - The extent pointed to by 'path' contains a superset
3501 * of the logical span [map->m_lblk, map->m_lblk + map->m_len).
3502 *
3503 * Post-conditions on success:
3504 * - the returned value is the number of blocks beyond map->l_lblk
3505 * that are allocated and initialized.
3506 * It is guaranteed to be >= map->m_len.
Amit Arora56055d32007-07-17 21:42:38 -04003507 */
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -05003508static int ext4_ext_convert_to_initialized(handle_t *handle,
Theodore Ts'oe35fd662010-05-16 19:00:00 -04003509 struct inode *inode,
3510 struct ext4_map_blocks *map,
Theodore Ts'odfe50802014-09-01 14:37:09 -04003511 struct ext4_ext_path **ppath,
Lukas Czerner27dd4382013-04-09 22:11:22 -04003512 int flags)
Amit Arora56055d32007-07-17 21:42:38 -04003513{
Theodore Ts'odfe50802014-09-01 14:37:09 -04003514 struct ext4_ext_path *path = *ppath;
Zheng Liu67a5da52012-08-17 09:54:17 -04003515 struct ext4_sb_info *sbi;
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003516 struct ext4_extent_header *eh;
Yongqiang Yang667eff32011-05-03 12:25:07 -04003517 struct ext4_map_blocks split_map;
Jan Kara4f8caa62017-05-26 17:40:52 -04003518 struct ext4_extent zero_ex1, zero_ex2;
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003519 struct ext4_extent *ex, *abut_ex;
Dmitry Monakhov21ca087a2010-05-16 06:00:00 -04003520 ext4_lblk_t ee_block, eof_block;
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003521 unsigned int ee_len, depth, map_len = map->m_len;
3522 int allocated = 0, max_zeroout = 0;
Amit Arora56055d32007-07-17 21:42:38 -04003523 int err = 0;
Jan Kara4f8caa62017-05-26 17:40:52 -04003524 int split_flag = EXT4_EXT_DATA_VALID2;
Dmitry Monakhov21ca087a2010-05-16 06:00:00 -04003525
3526 ext_debug("ext4_ext_convert_to_initialized: inode %lu, logical"
3527 "block %llu, max_blocks %u\n", inode->i_ino,
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003528 (unsigned long long)map->m_lblk, map_len);
Dmitry Monakhov21ca087a2010-05-16 06:00:00 -04003529
Zheng Liu67a5da52012-08-17 09:54:17 -04003530 sbi = EXT4_SB(inode->i_sb);
Dmitry Monakhov21ca087a2010-05-16 06:00:00 -04003531 eof_block = (inode->i_size + inode->i_sb->s_blocksize - 1) >>
3532 inode->i_sb->s_blocksize_bits;
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003533 if (eof_block < map->m_lblk + map_len)
3534 eof_block = map->m_lblk + map_len;
Amit Arora56055d32007-07-17 21:42:38 -04003535
3536 depth = ext_depth(inode);
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003537 eh = path[depth].p_hdr;
Amit Arora56055d32007-07-17 21:42:38 -04003538 ex = path[depth].p_ext;
3539 ee_block = le32_to_cpu(ex->ee_block);
3540 ee_len = ext4_ext_get_actual_len(ex);
Jan Kara4f8caa62017-05-26 17:40:52 -04003541 zero_ex1.ee_len = 0;
3542 zero_ex2.ee_len = 0;
Dmitry Monakhov21ca087a2010-05-16 06:00:00 -04003543
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003544 trace_ext4_ext_convert_to_initialized_enter(inode, map, ex);
3545
3546 /* Pre-conditions */
Lukas Czerner556615d2014-04-20 23:45:47 -04003547 BUG_ON(!ext4_ext_is_unwritten(ex));
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003548 BUG_ON(!in_range(map->m_lblk, ee_block, ee_len));
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003549
3550 /*
3551 * Attempt to transfer newly initialized blocks from the currently
Lukas Czerner556615d2014-04-20 23:45:47 -04003552 * unwritten extent to its neighbor. This is much cheaper
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003553 * than an insertion followed by a merge as those involve costly
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003554 * memmove() calls. Transferring to the left is the common case in
3555 * steady state for workloads doing fallocate(FALLOC_FL_KEEP_SIZE)
3556 * followed by append writes.
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003557 *
3558 * Limitations of the current logic:
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003559 * - L1: we do not deal with writes covering the whole extent.
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003560 * This would require removing the extent if the transfer
3561 * is possible.
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003562 * - L2: we only attempt to merge with an extent stored in the
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003563 * same extent tree node.
3564 */
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003565 if ((map->m_lblk == ee_block) &&
3566 /* See if we can merge left */
3567 (map_len < ee_len) && /*L1*/
3568 (ex > EXT_FIRST_EXTENT(eh))) { /*L2*/
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003569 ext4_lblk_t prev_lblk;
3570 ext4_fsblk_t prev_pblk, ee_pblk;
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003571 unsigned int prev_len;
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003572
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003573 abut_ex = ex - 1;
3574 prev_lblk = le32_to_cpu(abut_ex->ee_block);
3575 prev_len = ext4_ext_get_actual_len(abut_ex);
3576 prev_pblk = ext4_ext_pblock(abut_ex);
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003577 ee_pblk = ext4_ext_pblock(ex);
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003578
3579 /*
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003580 * A transfer of blocks from 'ex' to 'abut_ex' is allowed
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003581 * upon those conditions:
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003582 * - C1: abut_ex is initialized,
3583 * - C2: abut_ex is logically abutting ex,
3584 * - C3: abut_ex is physically abutting ex,
3585 * - C4: abut_ex can receive the additional blocks without
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003586 * overflowing the (initialized) length limit.
3587 */
Lukas Czerner556615d2014-04-20 23:45:47 -04003588 if ((!ext4_ext_is_unwritten(abut_ex)) && /*C1*/
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003589 ((prev_lblk + prev_len) == ee_block) && /*C2*/
3590 ((prev_pblk + prev_len) == ee_pblk) && /*C3*/
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003591 (prev_len < (EXT_INIT_MAX_LEN - map_len))) { /*C4*/
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003592 err = ext4_ext_get_access(handle, inode, path + depth);
3593 if (err)
3594 goto out;
3595
3596 trace_ext4_ext_convert_to_initialized_fastpath(inode,
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003597 map, ex, abut_ex);
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003598
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003599 /* Shift the start of ex by 'map_len' blocks */
3600 ex->ee_block = cpu_to_le32(ee_block + map_len);
3601 ext4_ext_store_pblock(ex, ee_pblk + map_len);
3602 ex->ee_len = cpu_to_le16(ee_len - map_len);
Lukas Czerner556615d2014-04-20 23:45:47 -04003603 ext4_ext_mark_unwritten(ex); /* Restore the flag */
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003604
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003605 /* Extend abut_ex by 'map_len' blocks */
3606 abut_ex->ee_len = cpu_to_le16(prev_len + map_len);
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003607
3608 /* Result: number of initialized blocks past m_lblk */
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003609 allocated = map_len;
3610 }
3611 } else if (((map->m_lblk + map_len) == (ee_block + ee_len)) &&
3612 (map_len < ee_len) && /*L1*/
3613 ex < EXT_LAST_EXTENT(eh)) { /*L2*/
3614 /* See if we can merge right */
3615 ext4_lblk_t next_lblk;
3616 ext4_fsblk_t next_pblk, ee_pblk;
3617 unsigned int next_len;
3618
3619 abut_ex = ex + 1;
3620 next_lblk = le32_to_cpu(abut_ex->ee_block);
3621 next_len = ext4_ext_get_actual_len(abut_ex);
3622 next_pblk = ext4_ext_pblock(abut_ex);
3623 ee_pblk = ext4_ext_pblock(ex);
3624
3625 /*
3626 * A transfer of blocks from 'ex' to 'abut_ex' is allowed
3627 * upon those conditions:
3628 * - C1: abut_ex is initialized,
3629 * - C2: abut_ex is logically abutting ex,
3630 * - C3: abut_ex is physically abutting ex,
3631 * - C4: abut_ex can receive the additional blocks without
3632 * overflowing the (initialized) length limit.
3633 */
Lukas Czerner556615d2014-04-20 23:45:47 -04003634 if ((!ext4_ext_is_unwritten(abut_ex)) && /*C1*/
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003635 ((map->m_lblk + map_len) == next_lblk) && /*C2*/
3636 ((ee_pblk + ee_len) == next_pblk) && /*C3*/
3637 (next_len < (EXT_INIT_MAX_LEN - map_len))) { /*C4*/
3638 err = ext4_ext_get_access(handle, inode, path + depth);
3639 if (err)
3640 goto out;
3641
3642 trace_ext4_ext_convert_to_initialized_fastpath(inode,
3643 map, ex, abut_ex);
3644
3645 /* Shift the start of abut_ex by 'map_len' blocks */
3646 abut_ex->ee_block = cpu_to_le32(next_lblk - map_len);
3647 ext4_ext_store_pblock(abut_ex, next_pblk - map_len);
3648 ex->ee_len = cpu_to_le16(ee_len - map_len);
Lukas Czerner556615d2014-04-20 23:45:47 -04003649 ext4_ext_mark_unwritten(ex); /* Restore the flag */
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003650
3651 /* Extend abut_ex by 'map_len' blocks */
3652 abut_ex->ee_len = cpu_to_le16(next_len + map_len);
3653
3654 /* Result: number of initialized blocks past m_lblk */
3655 allocated = map_len;
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003656 }
3657 }
Lukas Czernerbc2d9db2013-04-03 23:33:27 -04003658 if (allocated) {
3659 /* Mark the block containing both extents as dirty */
3660 ext4_ext_dirty(handle, inode, path + depth);
3661
3662 /* Update path to point to the right extent */
3663 path[depth].p_ext = abut_ex;
3664 goto out;
3665 } else
3666 allocated = ee_len - (map->m_lblk - ee_block);
Eric Gouriou6f91bc52011-10-27 11:43:23 -04003667
Yongqiang Yang667eff32011-05-03 12:25:07 -04003668 WARN_ON(map->m_lblk < ee_block);
Dmitry Monakhov21ca087a2010-05-16 06:00:00 -04003669 /*
3670 * It is safe to convert extent to initialized via explicit
Yongqiang Yang9e740562014-01-06 14:05:23 -05003671 * zeroout only if extent is fully inside i_size or new_size.
Dmitry Monakhov21ca087a2010-05-16 06:00:00 -04003672 */
Yongqiang Yang667eff32011-05-03 12:25:07 -04003673 split_flag |= ee_block + ee_len <= eof_block ? EXT4_EXT_MAY_ZEROOUT : 0;
Dmitry Monakhov21ca087a2010-05-16 06:00:00 -04003674
Zheng Liu67a5da52012-08-17 09:54:17 -04003675 if (EXT4_EXT_MAY_ZEROOUT & split_flag)
3676 max_zeroout = sbi->s_extent_max_zeroout_kb >>
Lukas Czerner4f42f802013-03-12 12:40:04 -04003677 (inode->i_sb->s_blocksize_bits - 10);
Zheng Liu67a5da52012-08-17 09:54:17 -04003678
Amit Arora56055d32007-07-17 21:42:38 -04003679 /*
Jan Kara4f8caa62017-05-26 17:40:52 -04003680 * five cases:
Yongqiang Yang667eff32011-05-03 12:25:07 -04003681 * 1. split the extent into three extents.
Jan Kara4f8caa62017-05-26 17:40:52 -04003682 * 2. split the extent into two extents, zeroout the head of the first
3683 * extent.
3684 * 3. split the extent into two extents, zeroout the tail of the second
3685 * extent.
Yongqiang Yang667eff32011-05-03 12:25:07 -04003686 * 4. split the extent into two extents with out zeroout.
Jan Kara4f8caa62017-05-26 17:40:52 -04003687 * 5. no splitting needed, just possibly zeroout the head and / or the
3688 * tail of the extent.
Amit Arora56055d32007-07-17 21:42:38 -04003689 */
Yongqiang Yang667eff32011-05-03 12:25:07 -04003690 split_map.m_lblk = map->m_lblk;
3691 split_map.m_len = map->m_len;
3692
Jan Kara4f8caa62017-05-26 17:40:52 -04003693 if (max_zeroout && (allocated > split_map.m_len)) {
Zheng Liu67a5da52012-08-17 09:54:17 -04003694 if (allocated <= max_zeroout) {
Jan Kara4f8caa62017-05-26 17:40:52 -04003695 /* case 3 or 5 */
3696 zero_ex1.ee_block =
3697 cpu_to_le32(split_map.m_lblk +
3698 split_map.m_len);
3699 zero_ex1.ee_len =
3700 cpu_to_le16(allocated - split_map.m_len);
3701 ext4_ext_store_pblock(&zero_ex1,
3702 ext4_ext_pblock(ex) + split_map.m_lblk +
3703 split_map.m_len - ee_block);
3704 err = ext4_ext_zeroout(inode, &zero_ex1);
Amit Arora56055d32007-07-17 21:42:38 -04003705 if (err)
3706 goto out;
Yongqiang Yang667eff32011-05-03 12:25:07 -04003707 split_map.m_len = allocated;
Jan Kara4f8caa62017-05-26 17:40:52 -04003708 }
3709 if (split_map.m_lblk - ee_block + split_map.m_len <
3710 max_zeroout) {
3711 /* case 2 or 5 */
3712 if (split_map.m_lblk != ee_block) {
3713 zero_ex2.ee_block = ex->ee_block;
3714 zero_ex2.ee_len = cpu_to_le16(split_map.m_lblk -
Yongqiang Yang667eff32011-05-03 12:25:07 -04003715 ee_block);
Jan Kara4f8caa62017-05-26 17:40:52 -04003716 ext4_ext_store_pblock(&zero_ex2,
Yongqiang Yang667eff32011-05-03 12:25:07 -04003717 ext4_ext_pblock(ex));
Jan Kara4f8caa62017-05-26 17:40:52 -04003718 err = ext4_ext_zeroout(inode, &zero_ex2);
Yongqiang Yang667eff32011-05-03 12:25:07 -04003719 if (err)
3720 goto out;
3721 }
3722
Jan Kara4f8caa62017-05-26 17:40:52 -04003723 split_map.m_len += split_map.m_lblk - ee_block;
Yongqiang Yang667eff32011-05-03 12:25:07 -04003724 split_map.m_lblk = ee_block;
Allison Henderson9b940f82011-05-16 10:11:09 -04003725 allocated = map->m_len;
Amit Arora56055d32007-07-17 21:42:38 -04003726 }
3727 }
Yongqiang Yang667eff32011-05-03 12:25:07 -04003728
Jan Karaae9e9c6a2014-10-30 10:53:17 -04003729 err = ext4_split_extent(handle, inode, ppath, &split_map, split_flag,
3730 flags);
3731 if (err > 0)
3732 err = 0;
Amit Arora56055d32007-07-17 21:42:38 -04003733out:
Zheng Liuadb23552013-03-10 21:13:05 -04003734 /* If we have gotten a failure, don't zero out status tree */
Jan Kara4f8caa62017-05-26 17:40:52 -04003735 if (!err) {
3736 err = ext4_zeroout_es(inode, &zero_ex1);
3737 if (!err)
3738 err = ext4_zeroout_es(inode, &zero_ex2);
3739 }
Amit Arora56055d32007-07-17 21:42:38 -04003740 return err ? err : allocated;
3741}
3742
Aneesh Kumar K.Vc278bfe2008-01-28 23:58:27 -05003743/*
Theodore Ts'oe35fd662010-05-16 19:00:00 -04003744 * This function is called by ext4_ext_map_blocks() from
Mingming Cao00314622009-09-28 15:49:08 -04003745 * ext4_get_blocks_dio_write() when DIO to write
Lukas Czerner556615d2014-04-20 23:45:47 -04003746 * to an unwritten extent.
Mingming Cao00314622009-09-28 15:49:08 -04003747 *
Lukas Czerner556615d2014-04-20 23:45:47 -04003748 * Writing to an unwritten extent may result in splitting the unwritten
3749 * extent into multiple initialized/unwritten extents (up to three)
Mingming Cao00314622009-09-28 15:49:08 -04003750 * There are three possibilities:
Lukas Czerner556615d2014-04-20 23:45:47 -04003751 * a> There is no split required: Entire extent should be unwritten
Mingming Cao00314622009-09-28 15:49:08 -04003752 * b> Splits in two extents: Write is happening at either end of the extent
3753 * c> Splits in three extents: Somone is writing in middle of the extent
3754 *
Lukas Czernerb8a86842014-03-18 18:05:35 -04003755 * This works the same way in the case of initialized -> unwritten conversion.
3756 *
Mingming Cao00314622009-09-28 15:49:08 -04003757 * One of more index blocks maybe needed if the extent tree grow after
Lukas Czerner556615d2014-04-20 23:45:47 -04003758 * the unwritten extent split. To prevent ENOSPC occur at the IO
3759 * complete, we need to split the unwritten extent before DIO submit
3760 * the IO. The unwritten extent called at this time will be split
3761 * into three unwritten extent(at most). After IO complete, the part
Mingming Cao00314622009-09-28 15:49:08 -04003762 * being filled will be convert to initialized by the end_io callback function
3763 * via ext4_convert_unwritten_extents().
Mingmingba230c32009-11-06 04:01:23 -05003764 *
Lukas Czerner556615d2014-04-20 23:45:47 -04003765 * Returns the size of unwritten extent to be written on success.
Mingming Cao00314622009-09-28 15:49:08 -04003766 */
Lukas Czernerb8a86842014-03-18 18:05:35 -04003767static int ext4_split_convert_extents(handle_t *handle,
Mingming Cao00314622009-09-28 15:49:08 -04003768 struct inode *inode,
Theodore Ts'oe35fd662010-05-16 19:00:00 -04003769 struct ext4_map_blocks *map,
Theodore Ts'odfe50802014-09-01 14:37:09 -04003770 struct ext4_ext_path **ppath,
Mingming Cao00314622009-09-28 15:49:08 -04003771 int flags)
3772{
Theodore Ts'odfe50802014-09-01 14:37:09 -04003773 struct ext4_ext_path *path = *ppath;
Yongqiang Yang667eff32011-05-03 12:25:07 -04003774 ext4_lblk_t eof_block;
3775 ext4_lblk_t ee_block;
3776 struct ext4_extent *ex;
3777 unsigned int ee_len;
3778 int split_flag = 0, depth;
Mingming Cao00314622009-09-28 15:49:08 -04003779
Lukas Czernerb8a86842014-03-18 18:05:35 -04003780 ext_debug("%s: inode %lu, logical block %llu, max_blocks %u\n",
3781 __func__, inode->i_ino,
3782 (unsigned long long)map->m_lblk, map->m_len);
Dmitry Monakhov21ca087a2010-05-16 06:00:00 -04003783
3784 eof_block = (inode->i_size + inode->i_sb->s_blocksize - 1) >>
3785 inode->i_sb->s_blocksize_bits;
Theodore Ts'oe35fd662010-05-16 19:00:00 -04003786 if (eof_block < map->m_lblk + map->m_len)
3787 eof_block = map->m_lblk + map->m_len;
Mingming Cao00314622009-09-28 15:49:08 -04003788 /*
Dmitry Monakhov21ca087a2010-05-16 06:00:00 -04003789 * It is safe to convert extent to initialized via explicit
3790 * zeroout only if extent is fully insde i_size or new_size.
3791 */
Yongqiang Yang667eff32011-05-03 12:25:07 -04003792 depth = ext_depth(inode);
3793 ex = path[depth].p_ext;
3794 ee_block = le32_to_cpu(ex->ee_block);
3795 ee_len = ext4_ext_get_actual_len(ex);
Dmitry Monakhov21ca087a2010-05-16 06:00:00 -04003796
Lukas Czernerb8a86842014-03-18 18:05:35 -04003797 /* Convert to unwritten */
3798 if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) {
3799 split_flag |= EXT4_EXT_DATA_VALID1;
3800 /* Convert to initialized */
3801 } else if (flags & EXT4_GET_BLOCKS_CONVERT) {
3802 split_flag |= ee_block + ee_len <= eof_block ?
3803 EXT4_EXT_MAY_ZEROOUT : 0;
Lukas Czerner556615d2014-04-20 23:45:47 -04003804 split_flag |= (EXT4_EXT_MARK_UNWRIT2 | EXT4_EXT_DATA_VALID2);
Lukas Czernerb8a86842014-03-18 18:05:35 -04003805 }
Yongqiang Yang667eff32011-05-03 12:25:07 -04003806 flags |= EXT4_GET_BLOCKS_PRE_IO;
Theodore Ts'odfe50802014-09-01 14:37:09 -04003807 return ext4_split_extent(handle, inode, ppath, map, split_flag, flags);
Mingming Cao00314622009-09-28 15:49:08 -04003808}
Yongqiang Yang197217a2011-05-03 11:45:29 -04003809
Jiaying Zhangc7064ef2010-03-02 13:28:44 -05003810static int ext4_convert_unwritten_extents_endio(handle_t *handle,
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003811 struct inode *inode,
3812 struct ext4_map_blocks *map,
Theodore Ts'odfe50802014-09-01 14:37:09 -04003813 struct ext4_ext_path **ppath)
Mingming Cao00314622009-09-28 15:49:08 -04003814{
Theodore Ts'odfe50802014-09-01 14:37:09 -04003815 struct ext4_ext_path *path = *ppath;
Mingming Cao00314622009-09-28 15:49:08 -04003816 struct ext4_extent *ex;
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003817 ext4_lblk_t ee_block;
3818 unsigned int ee_len;
Mingming Cao00314622009-09-28 15:49:08 -04003819 int depth;
3820 int err = 0;
Mingming Cao00314622009-09-28 15:49:08 -04003821
3822 depth = ext_depth(inode);
Mingming Cao00314622009-09-28 15:49:08 -04003823 ex = path[depth].p_ext;
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003824 ee_block = le32_to_cpu(ex->ee_block);
3825 ee_len = ext4_ext_get_actual_len(ex);
Mingming Cao00314622009-09-28 15:49:08 -04003826
Yongqiang Yang197217a2011-05-03 11:45:29 -04003827 ext_debug("ext4_convert_unwritten_extents_endio: inode %lu, logical"
3828 "block %llu, max_blocks %u\n", inode->i_ino,
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003829 (unsigned long long)ee_block, ee_len);
3830
Dmitry Monakhovff95ec22013-03-04 00:41:05 -05003831 /* If extent is larger than requested it is a clear sign that we still
3832 * have some extent state machine issues left. So extent_split is still
3833 * required.
3834 * TODO: Once all related issues will be fixed this situation should be
3835 * illegal.
3836 */
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003837 if (ee_block != map->m_lblk || ee_len > map->m_len) {
Rakesh Pandite3d550c2019-08-22 22:53:46 -04003838#ifdef CONFIG_EXT4_DEBUG
3839 ext4_warning(inode->i_sb, "Inode (%ld) finished: extent logical block %llu,"
Jakub Wilk8d2ae1c2016-04-27 01:11:21 -04003840 " len %u; IO logical block %llu, len %u",
Dmitry Monakhovff95ec22013-03-04 00:41:05 -05003841 inode->i_ino, (unsigned long long)ee_block, ee_len,
3842 (unsigned long long)map->m_lblk, map->m_len);
3843#endif
Theodore Ts'odfe50802014-09-01 14:37:09 -04003844 err = ext4_split_convert_extents(handle, inode, map, ppath,
Lukas Czernerb8a86842014-03-18 18:05:35 -04003845 EXT4_GET_BLOCKS_CONVERT);
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003846 if (err < 0)
Theodore Ts'odfe50802014-09-01 14:37:09 -04003847 return err;
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04003848 path = ext4_find_extent(inode, map->m_lblk, ppath, 0);
Theodore Ts'odfe50802014-09-01 14:37:09 -04003849 if (IS_ERR(path))
3850 return PTR_ERR(path);
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04003851 depth = ext_depth(inode);
3852 ex = path[depth].p_ext;
3853 }
Yongqiang Yang197217a2011-05-03 11:45:29 -04003854
Mingming Cao00314622009-09-28 15:49:08 -04003855 err = ext4_ext_get_access(handle, inode, path + depth);
3856 if (err)
3857 goto out;
3858 /* first mark the extent as initialized */
3859 ext4_ext_mark_initialized(ex);
3860
Yongqiang Yang197217a2011-05-03 11:45:29 -04003861 /* note: ext4_ext_correct_indexes() isn't needed here because
3862 * borders are not changed
Mingming Cao00314622009-09-28 15:49:08 -04003863 */
Theodore Ts'oecb94f52012-08-17 09:44:17 -04003864 ext4_ext_try_to_merge(handle, inode, path, ex);
Yongqiang Yang197217a2011-05-03 11:45:29 -04003865
Mingming Cao00314622009-09-28 15:49:08 -04003866 /* Mark modified extent as dirty */
Theodore Ts'oecb94f52012-08-17 09:44:17 -04003867 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
Mingming Cao00314622009-09-28 15:49:08 -04003868out:
3869 ext4_ext_show_leaf(inode, path);
3870 return err;
3871}
3872
Theodore Ts'o58590b02010-10-27 21:23:12 -04003873/*
3874 * Handle EOFBLOCKS_FL flag, clearing it if necessary
3875 */
3876static int check_eofblocks_fl(handle_t *handle, struct inode *inode,
Eric Sandeend002ebf2011-01-10 13:03:35 -05003877 ext4_lblk_t lblk,
Theodore Ts'o58590b02010-10-27 21:23:12 -04003878 struct ext4_ext_path *path,
3879 unsigned int len)
3880{
3881 int i, depth;
3882 struct ext4_extent_header *eh;
Sergey Senozhatsky65922cb2011-03-23 14:08:27 -04003883 struct ext4_extent *last_ex;
Theodore Ts'o58590b02010-10-27 21:23:12 -04003884
3885 if (!ext4_test_inode_flag(inode, EXT4_INODE_EOFBLOCKS))
3886 return 0;
3887
3888 depth = ext_depth(inode);
3889 eh = path[depth].p_hdr;
Theodore Ts'o58590b02010-10-27 21:23:12 -04003890
Lukas Czernerafcff5d2012-03-21 21:47:55 -04003891 /*
3892 * We're going to remove EOFBLOCKS_FL entirely in future so we
3893 * do not care for this case anymore. Simply remove the flag
3894 * if there are no extents.
3895 */
3896 if (unlikely(!eh->eh_entries))
3897 goto out;
Theodore Ts'o58590b02010-10-27 21:23:12 -04003898 last_ex = EXT_LAST_EXTENT(eh);
3899 /*
3900 * We should clear the EOFBLOCKS_FL flag if we are writing the
3901 * last block in the last extent in the file. We test this by
3902 * first checking to see if the caller to
3903 * ext4_ext_get_blocks() was interested in the last block (or
3904 * a block beyond the last block) in the current extent. If
3905 * this turns out to be false, we can bail out from this
3906 * function immediately.
3907 */
Eric Sandeend002ebf2011-01-10 13:03:35 -05003908 if (lblk + len < le32_to_cpu(last_ex->ee_block) +
Theodore Ts'o58590b02010-10-27 21:23:12 -04003909 ext4_ext_get_actual_len(last_ex))
3910 return 0;
3911 /*
3912 * If the caller does appear to be planning to write at or
3913 * beyond the end of the current extent, we then test to see
3914 * if the current extent is the last extent in the file, by
3915 * checking to make sure it was reached via the rightmost node
3916 * at each level of the tree.
3917 */
3918 for (i = depth-1; i >= 0; i--)
3919 if (path[i].p_idx != EXT_LAST_INDEX(path[i].p_hdr))
3920 return 0;
Lukas Czernerafcff5d2012-03-21 21:47:55 -04003921out:
Theodore Ts'o58590b02010-10-27 21:23:12 -04003922 ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS);
3923 return ext4_mark_inode_dirty(handle, inode);
3924}
3925
Mingming Cao00314622009-09-28 15:49:08 -04003926static int
Theodore Ts'oe8b83d932014-09-01 14:35:09 -04003927convert_initialized_extent(handle_t *handle, struct inode *inode,
3928 struct ext4_map_blocks *map,
Eric Whitney29c6eaf2016-02-22 22:58:55 -05003929 struct ext4_ext_path **ppath,
Eryu Guan56263b42016-02-12 01:23:00 -05003930 unsigned int allocated)
Lukas Czernerb8a86842014-03-18 18:05:35 -04003931{
Theodore Ts'o4f224b82014-09-01 14:36:09 -04003932 struct ext4_ext_path *path = *ppath;
Theodore Ts'oe8b83d932014-09-01 14:35:09 -04003933 struct ext4_extent *ex;
3934 ext4_lblk_t ee_block;
3935 unsigned int ee_len;
3936 int depth;
Lukas Czernerb8a86842014-03-18 18:05:35 -04003937 int err = 0;
3938
3939 /*
3940 * Make sure that the extent is no bigger than we support with
Lukas Czerner556615d2014-04-20 23:45:47 -04003941 * unwritten extent
Lukas Czernerb8a86842014-03-18 18:05:35 -04003942 */
Lukas Czerner556615d2014-04-20 23:45:47 -04003943 if (map->m_len > EXT_UNWRITTEN_MAX_LEN)
3944 map->m_len = EXT_UNWRITTEN_MAX_LEN / 2;
Lukas Czernerb8a86842014-03-18 18:05:35 -04003945
Theodore Ts'oe8b83d932014-09-01 14:35:09 -04003946 depth = ext_depth(inode);
3947 ex = path[depth].p_ext;
3948 ee_block = le32_to_cpu(ex->ee_block);
3949 ee_len = ext4_ext_get_actual_len(ex);
3950
3951 ext_debug("%s: inode %lu, logical"
3952 "block %llu, max_blocks %u\n", __func__, inode->i_ino,
3953 (unsigned long long)ee_block, ee_len);
3954
3955 if (ee_block != map->m_lblk || ee_len > map->m_len) {
Theodore Ts'odfe50802014-09-01 14:37:09 -04003956 err = ext4_split_convert_extents(handle, inode, map, ppath,
Theodore Ts'oe8b83d932014-09-01 14:35:09 -04003957 EXT4_GET_BLOCKS_CONVERT_UNWRITTEN);
3958 if (err < 0)
3959 return err;
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04003960 path = ext4_find_extent(inode, map->m_lblk, ppath, 0);
Theodore Ts'oe8b83d932014-09-01 14:35:09 -04003961 if (IS_ERR(path))
3962 return PTR_ERR(path);
3963 depth = ext_depth(inode);
3964 ex = path[depth].p_ext;
3965 if (!ex) {
3966 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
3967 (unsigned long) map->m_lblk);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04003968 return -EFSCORRUPTED;
Theodore Ts'oe8b83d932014-09-01 14:35:09 -04003969 }
3970 }
3971
3972 err = ext4_ext_get_access(handle, inode, path + depth);
3973 if (err)
3974 return err;
3975 /* first mark the extent as unwritten */
3976 ext4_ext_mark_unwritten(ex);
3977
3978 /* note: ext4_ext_correct_indexes() isn't needed here because
3979 * borders are not changed
3980 */
3981 ext4_ext_try_to_merge(handle, inode, path, ex);
3982
3983 /* Mark modified extent as dirty */
3984 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3985 if (err)
3986 return err;
3987 ext4_ext_show_leaf(inode, path);
3988
3989 ext4_update_inode_fsync_trans(handle, inode, 1);
3990 err = check_eofblocks_fl(handle, inode, map->m_lblk, path, map->m_len);
3991 if (err)
3992 return err;
Lukas Czernerb8a86842014-03-18 18:05:35 -04003993 map->m_flags |= EXT4_MAP_UNWRITTEN;
3994 if (allocated > map->m_len)
3995 allocated = map->m_len;
3996 map->m_len = allocated;
Theodore Ts'oe8b83d932014-09-01 14:35:09 -04003997 return allocated;
Lukas Czernerb8a86842014-03-18 18:05:35 -04003998}
3999
4000static int
Lukas Czerner556615d2014-04-20 23:45:47 -04004001ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode,
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004002 struct ext4_map_blocks *map,
Theodore Ts'odfe50802014-09-01 14:37:09 -04004003 struct ext4_ext_path **ppath, int flags,
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004004 unsigned int allocated, ext4_fsblk_t newblock)
Mingming Cao00314622009-09-28 15:49:08 -04004005{
Theodore Ts'odfe50802014-09-01 14:37:09 -04004006 struct ext4_ext_path *path = *ppath;
Mingming Cao00314622009-09-28 15:49:08 -04004007 int ret = 0;
4008 int err = 0;
4009
Lukas Czerner556615d2014-04-20 23:45:47 -04004010 ext_debug("ext4_ext_handle_unwritten_extents: inode %lu, logical "
Zheng Liu88635ca2011-12-28 19:00:25 -05004011 "block %llu, max_blocks %u, flags %x, allocated %u\n",
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004012 inode->i_ino, (unsigned long long)map->m_lblk, map->m_len,
Mingming Cao00314622009-09-28 15:49:08 -04004013 flags, allocated);
4014 ext4_ext_show_leaf(inode, path);
4015
Lukas Czerner27dd4382013-04-09 22:11:22 -04004016 /*
Lukas Czerner556615d2014-04-20 23:45:47 -04004017 * When writing into unwritten space, we should not fail to
Lukas Czerner27dd4382013-04-09 22:11:22 -04004018 * allocate metadata blocks for the new extent block if needed.
4019 */
4020 flags |= EXT4_GET_BLOCKS_METADATA_NOFAIL;
4021
Lukas Czerner556615d2014-04-20 23:45:47 -04004022 trace_ext4_ext_handle_unwritten_extents(inode, map, flags,
Zheng Liub5645532012-11-08 14:33:43 -05004023 allocated, newblock);
Aditya Kalid8990242011-09-09 19:18:51 -04004024
Jiaying Zhangc7064ef2010-03-02 13:28:44 -05004025 /* get_block() before submit the IO, split the extent */
Lukas Czernerc8b459f2014-05-12 12:55:07 -04004026 if (flags & EXT4_GET_BLOCKS_PRE_IO) {
Theodore Ts'odfe50802014-09-01 14:37:09 -04004027 ret = ext4_split_convert_extents(handle, inode, map, ppath,
4028 flags | EXT4_GET_BLOCKS_CONVERT);
Dmitry Monakhov82e54222012-09-28 23:36:25 -04004029 if (ret <= 0)
4030 goto out;
Zheng Liua25a4e12013-02-18 00:28:04 -05004031 map->m_flags |= EXT4_MAP_UNWRITTEN;
Mingming Cao00314622009-09-28 15:49:08 -04004032 goto out;
4033 }
Jiaying Zhangc7064ef2010-03-02 13:28:44 -05004034 /* IO end_io complete, convert the filled extent to written */
Lukas Czernerc8b459f2014-05-12 12:55:07 -04004035 if (flags & EXT4_GET_BLOCKS_CONVERT) {
Jan Karac86d8db2015-12-07 15:10:26 -05004036 if (flags & EXT4_GET_BLOCKS_ZERO) {
4037 if (allocated > map->m_len)
4038 allocated = map->m_len;
4039 err = ext4_issue_zeroout(inode, map->m_lblk, newblock,
4040 allocated);
4041 if (err < 0)
4042 goto out2;
4043 }
Dmitry Monakhovdee1f972012-10-10 01:04:58 -04004044 ret = ext4_convert_unwritten_extents_endio(handle, inode, map,
Theodore Ts'odfe50802014-09-01 14:37:09 -04004045 ppath);
Theodore Ts'o58590b02010-10-27 21:23:12 -04004046 if (ret >= 0) {
Jan Karab436b9b2009-12-08 23:51:10 -05004047 ext4_update_inode_fsync_trans(handle, inode, 1);
Eric Sandeend002ebf2011-01-10 13:03:35 -05004048 err = check_eofblocks_fl(handle, inode, map->m_lblk,
4049 path, map->m_len);
Theodore Ts'o58590b02010-10-27 21:23:12 -04004050 } else
4051 err = ret;
Zheng Liucdee7842013-03-10 21:08:52 -04004052 map->m_flags |= EXT4_MAP_MAPPED;
Eric Whitney15cc1762014-02-12 10:42:45 -05004053 map->m_pblk = newblock;
Zheng Liucdee7842013-03-10 21:08:52 -04004054 if (allocated > map->m_len)
4055 allocated = map->m_len;
4056 map->m_len = allocated;
Mingming Cao00314622009-09-28 15:49:08 -04004057 goto out2;
4058 }
4059 /* buffered IO case */
4060 /*
4061 * repeat fallocate creation request
4062 * we already have an unwritten extent
4063 */
Lukas Czerner556615d2014-04-20 23:45:47 -04004064 if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) {
Zheng Liua25a4e12013-02-18 00:28:04 -05004065 map->m_flags |= EXT4_MAP_UNWRITTEN;
Mingming Cao00314622009-09-28 15:49:08 -04004066 goto map_out;
Zheng Liua25a4e12013-02-18 00:28:04 -05004067 }
Mingming Cao00314622009-09-28 15:49:08 -04004068
4069 /* buffered READ or buffered write_begin() lookup */
4070 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) {
4071 /*
4072 * We have blocks reserved already. We
4073 * return allocated blocks so that delalloc
4074 * won't do block reservation for us. But
4075 * the buffer head will be unmapped so that
4076 * a read from the block returns 0s.
4077 */
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004078 map->m_flags |= EXT4_MAP_UNWRITTEN;
Mingming Cao00314622009-09-28 15:49:08 -04004079 goto out1;
4080 }
4081
4082 /* buffered write, writepage time, convert*/
Theodore Ts'odfe50802014-09-01 14:37:09 -04004083 ret = ext4_ext_convert_to_initialized(handle, inode, map, ppath, flags);
Dmitry Monakhova4e5d882011-10-25 08:15:12 -04004084 if (ret >= 0)
Jan Karab436b9b2009-12-08 23:51:10 -05004085 ext4_update_inode_fsync_trans(handle, inode, 1);
Mingming Cao00314622009-09-28 15:49:08 -04004086out:
4087 if (ret <= 0) {
4088 err = ret;
4089 goto out2;
4090 } else
4091 allocated = ret;
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004092 map->m_flags |= EXT4_MAP_NEW;
zhangyi (F)16e08b12019-02-10 23:32:07 -05004093 if (allocated > map->m_len)
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004094 allocated = map->m_len;
Zheng Liu3a225672013-03-10 21:20:23 -04004095 map->m_len = allocated;
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -05004096
Mingming Cao00314622009-09-28 15:49:08 -04004097map_out:
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004098 map->m_flags |= EXT4_MAP_MAPPED;
Dmitry Monakhova4e5d882011-10-25 08:15:12 -04004099 if ((flags & EXT4_GET_BLOCKS_KEEP_SIZE) == 0) {
4100 err = check_eofblocks_fl(handle, inode, map->m_lblk, path,
4101 map->m_len);
4102 if (err < 0)
4103 goto out2;
4104 }
Mingming Cao00314622009-09-28 15:49:08 -04004105out1:
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004106 if (allocated > map->m_len)
4107 allocated = map->m_len;
Mingming Cao00314622009-09-28 15:49:08 -04004108 ext4_ext_show_leaf(inode, path);
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004109 map->m_pblk = newblock;
4110 map->m_len = allocated;
Mingming Cao00314622009-09-28 15:49:08 -04004111out2:
Mingming Cao00314622009-09-28 15:49:08 -04004112 return err ? err : allocated;
4113}
Theodore Ts'o58590b02010-10-27 21:23:12 -04004114
Mingming Cao00314622009-09-28 15:49:08 -04004115/*
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004116 * get_implied_cluster_alloc - check to see if the requested
4117 * allocation (in the map structure) overlaps with a cluster already
4118 * allocated in an extent.
Aditya Kalid8990242011-09-09 19:18:51 -04004119 * @sb The filesystem superblock structure
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004120 * @map The requested lblk->pblk mapping
4121 * @ex The extent structure which might contain an implied
4122 * cluster allocation
4123 *
4124 * This function is called by ext4_ext_map_blocks() after we failed to
4125 * find blocks that were already in the inode's extent tree. Hence,
4126 * we know that the beginning of the requested region cannot overlap
4127 * the extent from the inode's extent tree. There are three cases we
4128 * want to catch. The first is this case:
4129 *
4130 * |--- cluster # N--|
4131 * |--- extent ---| |---- requested region ---|
4132 * |==========|
4133 *
4134 * The second case that we need to test for is this one:
4135 *
4136 * |--------- cluster # N ----------------|
4137 * |--- requested region --| |------- extent ----|
4138 * |=======================|
4139 *
4140 * The third case is when the requested region lies between two extents
4141 * within the same cluster:
4142 * |------------- cluster # N-------------|
4143 * |----- ex -----| |---- ex_right ----|
4144 * |------ requested region ------|
4145 * |================|
4146 *
4147 * In each of the above cases, we need to set the map->m_pblk and
4148 * map->m_len so it corresponds to the return the extent labelled as
4149 * "|====|" from cluster #N, since it is already in use for data in
4150 * cluster EXT4_B2C(sbi, map->m_lblk). We will then return 1 to
4151 * signal to ext4_ext_map_blocks() that map->m_pblk should be treated
4152 * as a new "allocated" block region. Otherwise, we will return 0 and
4153 * ext4_ext_map_blocks() will then allocate one or more new clusters
4154 * by calling ext4_mb_new_blocks().
4155 */
Aditya Kalid8990242011-09-09 19:18:51 -04004156static int get_implied_cluster_alloc(struct super_block *sb,
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004157 struct ext4_map_blocks *map,
4158 struct ext4_extent *ex,
4159 struct ext4_ext_path *path)
4160{
Aditya Kalid8990242011-09-09 19:18:51 -04004161 struct ext4_sb_info *sbi = EXT4_SB(sb);
Theodore Ts'of5a44db2013-12-20 09:29:35 -05004162 ext4_lblk_t c_offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004163 ext4_lblk_t ex_cluster_start, ex_cluster_end;
Curt Wohlgemuth14d7f3e2011-12-18 17:39:02 -05004164 ext4_lblk_t rr_cluster_start;
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004165 ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block);
4166 ext4_fsblk_t ee_start = ext4_ext_pblock(ex);
4167 unsigned short ee_len = ext4_ext_get_actual_len(ex);
4168
4169 /* The extent passed in that we are trying to match */
4170 ex_cluster_start = EXT4_B2C(sbi, ee_block);
4171 ex_cluster_end = EXT4_B2C(sbi, ee_block + ee_len - 1);
4172
4173 /* The requested region passed into ext4_map_blocks() */
4174 rr_cluster_start = EXT4_B2C(sbi, map->m_lblk);
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004175
4176 if ((rr_cluster_start == ex_cluster_end) ||
4177 (rr_cluster_start == ex_cluster_start)) {
4178 if (rr_cluster_start == ex_cluster_end)
4179 ee_start += ee_len - 1;
Theodore Ts'of5a44db2013-12-20 09:29:35 -05004180 map->m_pblk = EXT4_PBLK_CMASK(sbi, ee_start) + c_offset;
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004181 map->m_len = min(map->m_len,
4182 (unsigned) sbi->s_cluster_ratio - c_offset);
4183 /*
4184 * Check for and handle this case:
4185 *
4186 * |--------- cluster # N-------------|
4187 * |------- extent ----|
4188 * |--- requested region ---|
4189 * |===========|
4190 */
4191
4192 if (map->m_lblk < ee_block)
4193 map->m_len = min(map->m_len, ee_block - map->m_lblk);
4194
4195 /*
4196 * Check for the case where there is already another allocated
4197 * block to the right of 'ex' but before the end of the cluster.
4198 *
4199 * |------------- cluster # N-------------|
4200 * |----- ex -----| |---- ex_right ----|
4201 * |------ requested region ------|
4202 * |================|
4203 */
4204 if (map->m_lblk > ee_block) {
4205 ext4_lblk_t next = ext4_ext_next_allocated_block(path);
4206 map->m_len = min(map->m_len, next - map->m_lblk);
4207 }
Aditya Kalid8990242011-09-09 19:18:51 -04004208
4209 trace_ext4_get_implied_cluster_alloc_exit(sb, map, 1);
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004210 return 1;
4211 }
Aditya Kalid8990242011-09-09 19:18:51 -04004212
4213 trace_ext4_get_implied_cluster_alloc_exit(sb, map, 0);
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004214 return 0;
4215}
4216
4217
4218/*
Mingming Caof5ab0d12008-02-25 15:29:55 -05004219 * Block allocation/map/preallocation routine for extents based files
4220 *
4221 *
Aneesh Kumar K.Vc278bfe2008-01-28 23:58:27 -05004222 * Need to be called with
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -05004223 * down_read(&EXT4_I(inode)->i_data_sem) if not allocating file system block
4224 * (ie, create is zero). Otherwise down_write(&EXT4_I(inode)->i_data_sem)
Mingming Caof5ab0d12008-02-25 15:29:55 -05004225 *
4226 * return > 0, number of of blocks already mapped/allocated
4227 * if create == 0 and these are pre-allocated blocks
4228 * buffer head is unmapped
4229 * otherwise blocks are mapped
4230 *
4231 * return = 0, if plain look up failed (blocks have not been allocated)
4232 * buffer head is unmapped
4233 *
4234 * return < 0, error case.
Aneesh Kumar K.Vc278bfe2008-01-28 23:58:27 -05004235 */
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004236int ext4_ext_map_blocks(handle_t *handle, struct inode *inode,
4237 struct ext4_map_blocks *map, int flags)
Alex Tomasa86c6182006-10-11 01:21:03 -07004238{
4239 struct ext4_ext_path *path = NULL;
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004240 struct ext4_extent newex, *ex, *ex2;
4241 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04004242 ext4_fsblk_t newblock = 0;
Eric Whitneyce37c422014-02-19 18:52:39 -05004243 int free_on_err = 0, err = 0, depth, ret;
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004244 unsigned int allocated = 0, offset = 0;
Yongqiang Yang81fdbb42011-10-29 09:23:38 -04004245 unsigned int allocated_clusters = 0;
Alex Tomasc9de5602008-01-29 00:19:52 -05004246 struct ext4_allocation_request ar;
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004247 ext4_lblk_t cluster_offset;
Jan Karacbd75842014-11-25 11:41:49 -05004248 bool map_from_cluster = false;
Alex Tomasa86c6182006-10-11 01:21:03 -07004249
Mingming84fe3be2009-09-01 08:44:37 -04004250 ext_debug("blocks %u/%u requested for inode %lu\n",
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004251 map->m_lblk, map->m_len, inode->i_ino);
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04004252 trace_ext4_ext_map_blocks_enter(inode, map->m_lblk, map->m_len, flags);
Alex Tomasa86c6182006-10-11 01:21:03 -07004253
Alex Tomasa86c6182006-10-11 01:21:03 -07004254 /* find extent for this block */
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04004255 path = ext4_find_extent(inode, map->m_lblk, NULL, 0);
Alex Tomasa86c6182006-10-11 01:21:03 -07004256 if (IS_ERR(path)) {
4257 err = PTR_ERR(path);
4258 path = NULL;
4259 goto out2;
4260 }
4261
4262 depth = ext_depth(inode);
4263
4264 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07004265 * consistent leaf must not be empty;
4266 * this situation is possible, though, _during_ tree modification;
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04004267 * this is why assert can't be put in ext4_find_extent()
Alex Tomasa86c6182006-10-11 01:21:03 -07004268 */
Frank Mayhar273df552010-03-02 11:46:09 -05004269 if (unlikely(path[depth].p_ext == NULL && depth != 0)) {
4270 EXT4_ERROR_INODE(inode, "bad extent address "
Theodore Ts'of70f3622010-05-16 23:00:00 -04004271 "lblock: %lu, depth: %d pblock %lld",
4272 (unsigned long) map->m_lblk, depth,
4273 path[depth].p_block);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004274 err = -EFSCORRUPTED;
Surbhi Palande034fb4c2009-12-14 09:53:52 -05004275 goto out2;
4276 }
Alex Tomasa86c6182006-10-11 01:21:03 -07004277
Avantika Mathur7e028972006-12-06 20:41:33 -08004278 ex = path[depth].p_ext;
4279 if (ex) {
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -05004280 ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block);
Theodore Ts'obf89d162010-10-27 21:30:14 -04004281 ext4_fsblk_t ee_start = ext4_ext_pblock(ex);
Amit Aroraa2df2a62007-07-17 21:42:41 -04004282 unsigned short ee_len;
Suparna Bhattacharya471d4012006-10-11 01:21:06 -07004283
Lukas Czernerb8a86842014-03-18 18:05:35 -04004284
Suparna Bhattacharya471d4012006-10-11 01:21:06 -07004285 /*
Lukas Czerner556615d2014-04-20 23:45:47 -04004286 * unwritten extents are treated as holes, except that
Amit Arora56055d32007-07-17 21:42:38 -04004287 * we split out initialized portions during a write.
Suparna Bhattacharya471d4012006-10-11 01:21:06 -07004288 */
Amit Aroraa2df2a62007-07-17 21:42:41 -04004289 ee_len = ext4_ext_get_actual_len(ex);
Aditya Kalid8990242011-09-09 19:18:51 -04004290
4291 trace_ext4_ext_show_extent(inode, ee_block, ee_start, ee_len);
4292
Randy Dunlapd0d856e2006-10-11 01:21:07 -07004293 /* if found extent covers block, simply return it */
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004294 if (in_range(map->m_lblk, ee_block, ee_len)) {
4295 newblock = map->m_lblk - ee_block + ee_start;
Randy Dunlapd0d856e2006-10-11 01:21:07 -07004296 /* number of remaining blocks in the extent */
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004297 allocated = ee_len - (map->m_lblk - ee_block);
4298 ext_debug("%u fit into %u:%d -> %llu\n", map->m_lblk,
4299 ee_block, ee_len, newblock);
Amit Arora56055d32007-07-17 21:42:38 -04004300
Lukas Czernerb8a86842014-03-18 18:05:35 -04004301 /*
4302 * If the extent is initialized check whether the
4303 * caller wants to convert it to unwritten.
4304 */
Lukas Czerner556615d2014-04-20 23:45:47 -04004305 if ((!ext4_ext_is_unwritten(ex)) &&
Lukas Czernerb8a86842014-03-18 18:05:35 -04004306 (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) {
Theodore Ts'oe8b83d932014-09-01 14:35:09 -04004307 allocated = convert_initialized_extent(
Theodore Ts'o4f224b82014-09-01 14:36:09 -04004308 handle, inode, map, &path,
Eric Whitney29c6eaf2016-02-22 22:58:55 -05004309 allocated);
Lukas Czernerb8a86842014-03-18 18:05:35 -04004310 goto out2;
Lukas Czerner556615d2014-04-20 23:45:47 -04004311 } else if (!ext4_ext_is_unwritten(ex))
Lukas Czerner78771912012-03-19 23:05:43 -04004312 goto out;
Zheng Liu69eb33d2013-02-18 00:31:07 -05004313
Lukas Czerner556615d2014-04-20 23:45:47 -04004314 ret = ext4_ext_handle_unwritten_extents(
Theodore Ts'odfe50802014-09-01 14:37:09 -04004315 handle, inode, map, &path, flags,
Lukas Czerner78771912012-03-19 23:05:43 -04004316 allocated, newblock);
Eric Whitneyce37c422014-02-19 18:52:39 -05004317 if (ret < 0)
4318 err = ret;
4319 else
4320 allocated = ret;
Eric Whitney31cf0f22014-03-13 23:14:46 -04004321 goto out2;
Alex Tomasa86c6182006-10-11 01:21:03 -07004322 }
4323 }
4324
4325 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07004326 * requested block isn't allocated yet;
Alex Tomasa86c6182006-10-11 01:21:03 -07004327 * we couldn't try to create block if create flag is zero
4328 */
Theodore Ts'oc2177052009-05-14 00:58:52 -04004329 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) {
Jan Kara140a5252016-03-09 22:46:57 -05004330 ext4_lblk_t hole_start, hole_len;
4331
Jan Karafacab4d2016-03-09 22:54:00 -05004332 hole_start = map->m_lblk;
4333 hole_len = ext4_ext_determine_hole(inode, path, &hole_start);
Amit Arora56055d32007-07-17 21:42:38 -04004334 /*
4335 * put just found gap into cache to speed up
4336 * subsequent requests
4337 */
Jan Kara140a5252016-03-09 22:46:57 -05004338 ext4_ext_put_gap_in_cache(inode, hole_start, hole_len);
Jan Karafacab4d2016-03-09 22:54:00 -05004339
4340 /* Update hole_len to reflect hole size after map->m_lblk */
4341 if (hole_start != map->m_lblk)
4342 hole_len -= map->m_lblk - hole_start;
4343 map->m_pblk = 0;
4344 map->m_len = min_t(unsigned int, map->m_len, hole_len);
4345
Alex Tomasa86c6182006-10-11 01:21:03 -07004346 goto out2;
4347 }
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004348
Alex Tomasa86c6182006-10-11 01:21:03 -07004349 /*
Theodore Ts'oc2ea3fd2008-10-10 09:40:52 -04004350 * Okay, we need to do block allocation.
Andrew Morton63f57932006-10-11 01:21:24 -07004351 */
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004352 newex.ee_block = cpu_to_le32(map->m_lblk);
Eric Whitneyd0abafa2014-01-06 14:00:23 -05004353 cluster_offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004354
4355 /*
4356 * If we are doing bigalloc, check to see if the extent returned
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04004357 * by ext4_find_extent() implies a cluster we can use.
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004358 */
4359 if (cluster_offset && ex &&
Aditya Kalid8990242011-09-09 19:18:51 -04004360 get_implied_cluster_alloc(inode->i_sb, map, ex, path)) {
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004361 ar.len = allocated = map->m_len;
4362 newblock = map->m_pblk;
Jan Karacbd75842014-11-25 11:41:49 -05004363 map_from_cluster = true;
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004364 goto got_allocated_blocks;
4365 }
Alex Tomasa86c6182006-10-11 01:21:03 -07004366
Alex Tomasc9de5602008-01-29 00:19:52 -05004367 /* find neighbour allocated blocks */
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004368 ar.lleft = map->m_lblk;
Alex Tomasc9de5602008-01-29 00:19:52 -05004369 err = ext4_ext_search_left(inode, path, &ar.lleft, &ar.pleft);
4370 if (err)
4371 goto out2;
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004372 ar.lright = map->m_lblk;
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004373 ex2 = NULL;
4374 err = ext4_ext_search_right(inode, path, &ar.lright, &ar.pright, &ex2);
Alex Tomasc9de5602008-01-29 00:19:52 -05004375 if (err)
4376 goto out2;
Amit Arora25d14f92007-05-24 13:04:13 -04004377
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004378 /* Check if the extent after searching to the right implies a
4379 * cluster we can use. */
4380 if ((sbi->s_cluster_ratio > 1) && ex2 &&
Aditya Kalid8990242011-09-09 19:18:51 -04004381 get_implied_cluster_alloc(inode->i_sb, map, ex2, path)) {
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004382 ar.len = allocated = map->m_len;
4383 newblock = map->m_pblk;
Jan Karacbd75842014-11-25 11:41:49 -05004384 map_from_cluster = true;
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004385 goto got_allocated_blocks;
4386 }
4387
Amit Arora749269f2007-07-18 09:02:56 -04004388 /*
4389 * See if request is beyond maximum number of blocks we can have in
4390 * a single extent. For an initialized extent this limit is
Lukas Czerner556615d2014-04-20 23:45:47 -04004391 * EXT_INIT_MAX_LEN and for an unwritten extent this limit is
4392 * EXT_UNWRITTEN_MAX_LEN.
Amit Arora749269f2007-07-18 09:02:56 -04004393 */
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004394 if (map->m_len > EXT_INIT_MAX_LEN &&
Lukas Czerner556615d2014-04-20 23:45:47 -04004395 !(flags & EXT4_GET_BLOCKS_UNWRIT_EXT))
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004396 map->m_len = EXT_INIT_MAX_LEN;
Lukas Czerner556615d2014-04-20 23:45:47 -04004397 else if (map->m_len > EXT_UNWRITTEN_MAX_LEN &&
4398 (flags & EXT4_GET_BLOCKS_UNWRIT_EXT))
4399 map->m_len = EXT_UNWRITTEN_MAX_LEN;
Amit Arora749269f2007-07-18 09:02:56 -04004400
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004401 /* Check if we can really insert (m_lblk)::(m_lblk + m_len) extent */
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004402 newex.ee_len = cpu_to_le16(map->m_len);
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004403 err = ext4_ext_check_overlap(sbi, inode, &newex, path);
Amit Arora25d14f92007-05-24 13:04:13 -04004404 if (err)
Aneesh Kumar K.Vb939e372008-01-28 23:58:27 -05004405 allocated = ext4_ext_get_actual_len(&newex);
Amit Arora25d14f92007-05-24 13:04:13 -04004406 else
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004407 allocated = map->m_len;
Alex Tomasc9de5602008-01-29 00:19:52 -05004408
4409 /* allocate new block */
4410 ar.inode = inode;
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004411 ar.goal = ext4_ext_find_goal(inode, path, map->m_lblk);
4412 ar.logical = map->m_lblk;
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004413 /*
4414 * We calculate the offset from the beginning of the cluster
4415 * for the logical block number, since when we allocate a
4416 * physical cluster, the physical block should start at the
4417 * same offset from the beginning of the cluster. This is
4418 * needed so that future calls to get_implied_cluster_alloc()
4419 * work correctly.
4420 */
Theodore Ts'of5a44db2013-12-20 09:29:35 -05004421 offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004422 ar.len = EXT4_NUM_B2C(sbi, offset+allocated);
4423 ar.goal -= offset;
4424 ar.logical -= offset;
Alex Tomasc9de5602008-01-29 00:19:52 -05004425 if (S_ISREG(inode->i_mode))
4426 ar.flags = EXT4_MB_HINT_DATA;
4427 else
4428 /* disable in-core preallocation for non-regular files */
4429 ar.flags = 0;
Vivek Haldar556b27a2011-05-25 07:41:54 -04004430 if (flags & EXT4_GET_BLOCKS_NO_NORMALIZE)
4431 ar.flags |= EXT4_MB_HINT_NOPREALLOC;
Theodore Ts'oe3cf5d52014-09-04 18:07:25 -04004432 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
4433 ar.flags |= EXT4_MB_DELALLOC_RESERVED;
Theodore Ts'oc5e298a2015-06-21 01:25:29 -04004434 if (flags & EXT4_GET_BLOCKS_METADATA_NOFAIL)
4435 ar.flags |= EXT4_MB_USE_RESERVED;
Alex Tomasc9de5602008-01-29 00:19:52 -05004436 newblock = ext4_mb_new_blocks(handle, &ar, &err);
Alex Tomasa86c6182006-10-11 01:21:03 -07004437 if (!newblock)
4438 goto out2;
Mingming84fe3be2009-09-01 08:44:37 -04004439 ext_debug("allocate new block: goal %llu, found %llu/%u\n",
Theodore Ts'o498e5f22008-11-05 00:14:04 -05004440 ar.goal, newblock, allocated);
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004441 free_on_err = 1;
Aditya Kali7b415bf2011-09-09 19:04:51 -04004442 allocated_clusters = ar.len;
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004443 ar.len = EXT4_C2B(sbi, ar.len) - offset;
4444 if (ar.len > allocated)
4445 ar.len = allocated;
Alex Tomasa86c6182006-10-11 01:21:03 -07004446
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004447got_allocated_blocks:
Alex Tomasa86c6182006-10-11 01:21:03 -07004448 /* try to insert new extent into found leaf and return */
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004449 ext4_ext_store_pblock(&newex, newblock + offset);
Alex Tomasc9de5602008-01-29 00:19:52 -05004450 newex.ee_len = cpu_to_le16(ar.len);
Lukas Czerner556615d2014-04-20 23:45:47 -04004451 /* Mark unwritten */
4452 if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT){
4453 ext4_ext_mark_unwritten(&newex);
Zheng Liua25a4e12013-02-18 00:28:04 -05004454 map->m_flags |= EXT4_MAP_UNWRITTEN;
Mingming Cao8d5d02e2009-09-28 15:48:29 -04004455 }
Jiaying Zhangc8d46e42010-02-24 09:52:53 -05004456
Dmitry Monakhova4e5d882011-10-25 08:15:12 -04004457 err = 0;
4458 if ((flags & EXT4_GET_BLOCKS_KEEP_SIZE) == 0)
4459 err = check_eofblocks_fl(handle, inode, map->m_lblk,
4460 path, ar.len);
Jiaying Zhang575a1d42011-07-10 20:07:25 -04004461 if (!err)
Theodore Ts'odfe50802014-09-01 14:37:09 -04004462 err = ext4_ext_insert_extent(handle, inode, &path,
Jiaying Zhang575a1d42011-07-10 20:07:25 -04004463 &newex, flags);
Dmitry Monakhov82e54222012-09-28 23:36:25 -04004464
Theodore Ts'o4d33b1e2011-09-09 18:52:51 -04004465 if (err && free_on_err) {
Maxim Patlasov7132de72011-07-10 19:37:48 -04004466 int fb_flags = flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE ?
4467 EXT4_FREE_BLOCKS_NO_QUOT_UPDATE : 0;
Alex Tomas315054f2007-05-24 13:04:25 -04004468 /* free data blocks we just allocated */
Alex Tomasc9de5602008-01-29 00:19:52 -05004469 /* not a good idea to call discard here directly,
4470 * but otherwise we'd need to call it every free() */
Theodore Ts'oc2ea3fd2008-10-10 09:40:52 -04004471 ext4_discard_preallocations(inode);
Theodore Ts'oc8e15132013-07-15 00:09:37 -04004472 ext4_free_blocks(handle, inode, NULL, newblock,
4473 EXT4_C2B(sbi, allocated_clusters), fb_flags);
Alex Tomasa86c6182006-10-11 01:21:03 -07004474 goto out2;
Alex Tomas315054f2007-05-24 13:04:25 -04004475 }
Alex Tomasa86c6182006-10-11 01:21:03 -07004476
Alex Tomasa86c6182006-10-11 01:21:03 -07004477 /* previous routine could use block we allocated */
Theodore Ts'obf89d162010-10-27 21:30:14 -04004478 newblock = ext4_ext_pblock(&newex);
Aneesh Kumar K.Vb939e372008-01-28 23:58:27 -05004479 allocated = ext4_ext_get_actual_len(&newex);
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004480 if (allocated > map->m_len)
4481 allocated = map->m_len;
4482 map->m_flags |= EXT4_MAP_NEW;
Alex Tomasa86c6182006-10-11 01:21:03 -07004483
Jan Karab436b9b2009-12-08 23:51:10 -05004484 /*
Eric Whitneyb6bf9172018-10-01 14:24:08 -04004485 * Reduce the reserved cluster count to reflect successful deferred
4486 * allocation of delayed allocated clusters or direct allocation of
4487 * clusters discovered to be delayed allocated. Once allocated, a
4488 * cluster is not included in the reserved count.
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -05004489 */
Eric Whitneyb6bf9172018-10-01 14:24:08 -04004490 if (test_opt(inode->i_sb, DELALLOC) && !map_from_cluster) {
4491 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) {
Lukas Czerner232ec872013-03-10 22:46:30 -04004492 /*
Eric Whitneyb6bf9172018-10-01 14:24:08 -04004493 * When allocating delayed allocated clusters, simply
4494 * reduce the reserved cluster count and claim quota
Lukas Czerner232ec872013-03-10 22:46:30 -04004495 */
4496 ext4_da_update_reserve_space(inode, allocated_clusters,
4497 1);
Eric Whitneyb6bf9172018-10-01 14:24:08 -04004498 } else {
4499 ext4_lblk_t lblk, len;
4500 unsigned int n;
4501
4502 /*
4503 * When allocating non-delayed allocated clusters
4504 * (from fallocate, filemap, DIO, or clusters
4505 * allocated when delalloc has been disabled by
4506 * ext4_nonda_switch), reduce the reserved cluster
4507 * count by the number of allocated clusters that
4508 * have previously been delayed allocated. Quota
4509 * has been claimed by ext4_mb_new_blocks() above,
4510 * so release the quota reservations made for any
4511 * previously delayed allocated clusters.
4512 */
4513 lblk = EXT4_LBLK_CMASK(sbi, map->m_lblk);
4514 len = allocated_clusters << sbi->s_cluster_bits;
4515 n = ext4_es_delayed_clu(inode, lblk, len);
4516 if (n > 0)
4517 ext4_da_update_reserve_space(inode, (int) n, 0);
Aditya Kali7b415bf2011-09-09 19:04:51 -04004518 }
4519 }
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -05004520
4521 /*
Jan Karab436b9b2009-12-08 23:51:10 -05004522 * Cache the extent and update transaction to commit on fdatasync only
Lukas Czerner556615d2014-04-20 23:45:47 -04004523 * when it is _not_ an unwritten extent.
Jan Karab436b9b2009-12-08 23:51:10 -05004524 */
Lukas Czerner556615d2014-04-20 23:45:47 -04004525 if ((flags & EXT4_GET_BLOCKS_UNWRIT_EXT) == 0)
Jan Karab436b9b2009-12-08 23:51:10 -05004526 ext4_update_inode_fsync_trans(handle, inode, 1);
Zheng Liu69eb33d2013-02-18 00:31:07 -05004527 else
Jan Karab436b9b2009-12-08 23:51:10 -05004528 ext4_update_inode_fsync_trans(handle, inode, 0);
Alex Tomasa86c6182006-10-11 01:21:03 -07004529out:
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004530 if (allocated > map->m_len)
4531 allocated = map->m_len;
Alex Tomasa86c6182006-10-11 01:21:03 -07004532 ext4_ext_show_leaf(inode, path);
Theodore Ts'oe35fd662010-05-16 19:00:00 -04004533 map->m_flags |= EXT4_MAP_MAPPED;
4534 map->m_pblk = newblock;
4535 map->m_len = allocated;
Alex Tomasa86c6182006-10-11 01:21:03 -07004536out2:
Theodore Ts'ob7ea89a2014-09-01 14:39:09 -04004537 ext4_ext_drop_refs(path);
4538 kfree(path);
Allison Hendersone8613042011-05-25 07:41:46 -04004539
Theodore Ts'o63b99962013-07-16 10:28:47 -04004540 trace_ext4_ext_map_blocks_exit(inode, flags, map,
4541 err ? err : allocated);
Lukas Czerner78771912012-03-19 23:05:43 -04004542 return err ? err : allocated;
Alex Tomasa86c6182006-10-11 01:21:03 -07004543}
4544
Theodore Ts'od0abb362016-11-13 22:02:28 -05004545int ext4_ext_truncate(handle_t *handle, struct inode *inode)
Alex Tomasa86c6182006-10-11 01:21:03 -07004546{
Alex Tomasa86c6182006-10-11 01:21:03 -07004547 struct super_block *sb = inode->i_sb;
Aneesh Kumar K.V725d26d2008-01-28 23:58:27 -05004548 ext4_lblk_t last_block;
Alex Tomasa86c6182006-10-11 01:21:03 -07004549 int err = 0;
4550
4551 /*
Randy Dunlapd0d856e2006-10-11 01:21:07 -07004552 * TODO: optimization is possible here.
4553 * Probably we need not scan at all,
4554 * because page truncation is enough.
Alex Tomasa86c6182006-10-11 01:21:03 -07004555 */
Alex Tomasa86c6182006-10-11 01:21:03 -07004556
4557 /* we have to know where to truncate from in crash case */
4558 EXT4_I(inode)->i_disksize = inode->i_size;
Theodore Ts'od0abb362016-11-13 22:02:28 -05004559 err = ext4_mark_inode_dirty(handle, inode);
4560 if (err)
4561 return err;
Alex Tomasa86c6182006-10-11 01:21:03 -07004562
4563 last_block = (inode->i_size + sb->s_blocksize - 1)
4564 >> EXT4_BLOCK_SIZE_BITS(sb);
Theodore Ts'o8acd5e92013-07-15 00:09:19 -04004565retry:
Zheng Liu51865fd2012-11-08 21:57:32 -05004566 err = ext4_es_remove_extent(inode, last_block,
4567 EXT_MAX_BLOCKS - last_block);
Theodore Ts'o94eec0f2013-07-29 12:12:56 -04004568 if (err == -ENOMEM) {
Theodore Ts'o8acd5e92013-07-15 00:09:19 -04004569 cond_resched();
4570 congestion_wait(BLK_RW_ASYNC, HZ/50);
4571 goto retry;
4572 }
Theodore Ts'od0abb362016-11-13 22:02:28 -05004573 if (err)
4574 return err;
4575 return ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1);
Alex Tomasa86c6182006-10-11 01:21:03 -07004576}
4577
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004578static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset,
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004579 ext4_lblk_t len, loff_t new_size,
Tahsin Erdogan77a2e842017-08-05 22:15:45 -04004580 int flags)
Amit Aroraa2df2a62007-07-17 21:42:41 -04004581{
Al Viro496ad9a2013-01-23 17:07:38 -05004582 struct inode *inode = file_inode(file);
Amit Aroraa2df2a62007-07-17 21:42:41 -04004583 handle_t *handle;
Amit Aroraa2df2a62007-07-17 21:42:41 -04004584 int ret = 0;
4585 int ret2 = 0;
4586 int retries = 0;
Lukas Czerner4134f5c2015-06-15 00:20:46 -04004587 int depth = 0;
Theodore Ts'o2ed88682010-05-16 20:00:00 -04004588 struct ext4_map_blocks map;
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004589 unsigned int credits;
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004590 loff_t epos;
Amit Aroraa2df2a62007-07-17 21:42:41 -04004591
Fabian Frederickc3fe4932016-09-15 11:52:07 -04004592 BUG_ON(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS));
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004593 map.m_lblk = offset;
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004594 map.m_len = len;
Greg Harm3c6fe772011-10-31 18:41:47 -04004595 /*
4596 * Don't normalize the request if it can fit in one extent so
4597 * that it doesn't get unnecessarily split into multiple
4598 * extents.
4599 */
Lukas Czerner556615d2014-04-20 23:45:47 -04004600 if (len <= EXT_UNWRITTEN_MAX_LEN)
Greg Harm3c6fe772011-10-31 18:41:47 -04004601 flags |= EXT4_GET_BLOCKS_NO_NORMALIZE;
Dmitry Monakhov60d46162012-10-05 11:32:02 -04004602
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004603 /*
4604 * credits to insert 1 extent into extent tree
4605 */
4606 credits = ext4_chunk_trans_blocks(inode, len);
Fabian Frederickc3fe4932016-09-15 11:52:07 -04004607 depth = ext_depth(inode);
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004608
Amit Aroraa2df2a62007-07-17 21:42:41 -04004609retry:
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004610 while (ret >= 0 && len) {
Lukas Czerner4134f5c2015-06-15 00:20:46 -04004611 /*
4612 * Recalculate credits when extent tree depth changes.
4613 */
Dan Carpenter011c88e2016-12-03 16:46:58 -05004614 if (depth != ext_depth(inode)) {
Lukas Czerner4134f5c2015-06-15 00:20:46 -04004615 credits = ext4_chunk_trans_blocks(inode, len);
4616 depth = ext_depth(inode);
4617 }
4618
Theodore Ts'o9924a922013-02-08 21:59:22 -05004619 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
4620 credits);
Amit Aroraa2df2a62007-07-17 21:42:41 -04004621 if (IS_ERR(handle)) {
4622 ret = PTR_ERR(handle);
4623 break;
4624 }
Dmitry Monakhova4e5d882011-10-25 08:15:12 -04004625 ret = ext4_map_blocks(handle, inode, &map, flags);
Aneesh Kumar K.V221879c2008-01-28 23:58:27 -05004626 if (ret <= 0) {
Lukas Czernerf282ac12014-03-18 17:44:35 -04004627 ext4_debug("inode #%lu: block %u: len %u: "
4628 "ext4_ext_map_blocks returned %d",
4629 inode->i_ino, map.m_lblk,
4630 map.m_len, ret);
Amit Aroraa2df2a62007-07-17 21:42:41 -04004631 ext4_mark_inode_dirty(handle, inode);
4632 ret2 = ext4_journal_stop(handle);
4633 break;
4634 }
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004635 map.m_lblk += ret;
4636 map.m_len = len = len - ret;
4637 epos = (loff_t)map.m_lblk << inode->i_blkbits;
Deepa Dinamanieeca7ea2016-11-14 21:40:10 -05004638 inode->i_ctime = current_time(inode);
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004639 if (new_size) {
4640 if (epos > new_size)
4641 epos = new_size;
4642 if (ext4_update_inode_size(inode, epos) & 0x1)
4643 inode->i_mtime = inode->i_ctime;
4644 } else {
4645 if (epos > inode->i_size)
4646 ext4_set_inode_flag(inode,
4647 EXT4_INODE_EOFBLOCKS);
4648 }
4649 ext4_mark_inode_dirty(handle, inode);
Eryu Guanc894aa92017-12-03 22:52:51 -05004650 ext4_update_inode_fsync_trans(handle, inode, 1);
Amit Aroraa2df2a62007-07-17 21:42:41 -04004651 ret2 = ext4_journal_stop(handle);
4652 if (ret2)
4653 break;
4654 }
Aneesh Kumar K.Vfd287842008-04-29 08:11:12 -04004655 if (ret == -ENOSPC &&
4656 ext4_should_retry_alloc(inode->i_sb, &retries)) {
4657 ret = 0;
Amit Aroraa2df2a62007-07-17 21:42:41 -04004658 goto retry;
Amit Aroraa2df2a62007-07-17 21:42:41 -04004659 }
Lukas Czernerf282ac12014-03-18 17:44:35 -04004660
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004661 return ret > 0 ? ret2 : ret;
4662}
4663
Eric Biggers43f81672019-12-31 12:04:40 -06004664static int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len);
4665
4666static int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len);
4667
Lukas Czernerb8a86842014-03-18 18:05:35 -04004668static long ext4_zero_range(struct file *file, loff_t offset,
4669 loff_t len, int mode)
4670{
4671 struct inode *inode = file_inode(file);
4672 handle_t *handle = NULL;
4673 unsigned int max_blocks;
4674 loff_t new_size = 0;
4675 int ret = 0;
4676 int flags;
Dmitry Monakhov69dc9532014-08-27 18:33:49 -04004677 int credits;
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004678 int partial_begin, partial_end;
Lukas Czernerb8a86842014-03-18 18:05:35 -04004679 loff_t start, end;
4680 ext4_lblk_t lblk;
Lukas Czernerb8a86842014-03-18 18:05:35 -04004681 unsigned int blkbits = inode->i_blkbits;
4682
4683 trace_ext4_zero_range(inode, offset, len, mode);
4684
Namjae Jeone1ee60f2014-05-27 12:48:55 -04004685 /* Call ext4_force_commit to flush all data in case of data=journal. */
4686 if (ext4_should_journal_data(inode)) {
4687 ret = ext4_force_commit(inode->i_sb);
4688 if (ret)
4689 return ret;
4690 }
4691
Lukas Czernerb8a86842014-03-18 18:05:35 -04004692 /*
Lukas Czernerb8a86842014-03-18 18:05:35 -04004693 * Round up offset. This is not fallocate, we neet to zero out
4694 * blocks, so convert interior block aligned part of the range to
4695 * unwritten and possibly manually zero out unaligned parts of the
4696 * range.
4697 */
4698 start = round_up(offset, 1 << blkbits);
4699 end = round_down((offset + len), 1 << blkbits);
4700
4701 if (start < offset || end > offset + len)
4702 return -EINVAL;
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004703 partial_begin = offset & ((1 << blkbits) - 1);
4704 partial_end = (offset + len) & ((1 << blkbits) - 1);
Lukas Czernerb8a86842014-03-18 18:05:35 -04004705
4706 lblk = start >> blkbits;
4707 max_blocks = (end >> blkbits);
4708 if (max_blocks < lblk)
4709 max_blocks = 0;
4710 else
4711 max_blocks -= lblk;
4712
Al Viro59551022016-01-22 15:40:57 -05004713 inode_lock(inode);
Lukas Czernerb8a86842014-03-18 18:05:35 -04004714
4715 /*
4716 * Indirect files do not support unwritten extnets
4717 */
4718 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
4719 ret = -EOPNOTSUPP;
4720 goto out_mutex;
4721 }
4722
4723 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
Eric Biggers9b02e492019-12-31 12:04:38 -06004724 (offset + len > inode->i_size ||
Theodore Ts'o51e3ae82017-10-06 23:09:55 -04004725 offset + len > EXT4_I(inode)->i_disksize)) {
Lukas Czernerb8a86842014-03-18 18:05:35 -04004726 new_size = offset + len;
4727 ret = inode_newsize_ok(inode, new_size);
4728 if (ret)
4729 goto out_mutex;
Lukas Czernerb8a86842014-03-18 18:05:35 -04004730 }
4731
Lukas Czerner0f2af212015-04-03 00:09:13 -04004732 flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
4733 if (mode & FALLOC_FL_KEEP_SIZE)
4734 flags |= EXT4_GET_BLOCKS_KEEP_SIZE;
4735
Jan Kara17048e82015-12-07 14:29:17 -05004736 /* Wait all existing dio workers, newcomers will block on i_mutex */
Jan Kara17048e82015-12-07 14:29:17 -05004737 inode_dio_wait(inode);
4738
Lukas Czerner0f2af212015-04-03 00:09:13 -04004739 /* Preallocate the range including the unaligned edges */
4740 if (partial_begin || partial_end) {
4741 ret = ext4_alloc_file_blocks(file,
4742 round_down(offset, 1 << blkbits) >> blkbits,
4743 (round_up((offset + len), 1 << blkbits) -
4744 round_down(offset, 1 << blkbits)) >> blkbits,
Tahsin Erdogan77a2e842017-08-05 22:15:45 -04004745 new_size, flags);
Lukas Czerner0f2af212015-04-03 00:09:13 -04004746 if (ret)
Nikolay Borisov1d398342018-03-22 11:52:10 -04004747 goto out_mutex;
Lukas Czerner0f2af212015-04-03 00:09:13 -04004748
4749 }
4750
4751 /* Zero range excluding the unaligned edges */
Lukas Czernerb8a86842014-03-18 18:05:35 -04004752 if (max_blocks > 0) {
Lukas Czerner0f2af212015-04-03 00:09:13 -04004753 flags |= (EXT4_GET_BLOCKS_CONVERT_UNWRITTEN |
4754 EXT4_EX_NOCACHE);
Lukas Czernerb8a86842014-03-18 18:05:35 -04004755
Jan Karaea3d7202015-12-07 14:28:03 -05004756 /*
4757 * Prevent page faults from reinstantiating pages we have
4758 * released from page cache.
4759 */
4760 down_write(&EXT4_I(inode)->i_mmap_sem);
Ross Zwisler430657b2018-07-29 17:00:22 -04004761
4762 ret = ext4_break_layouts(inode);
4763 if (ret) {
4764 up_write(&EXT4_I(inode)->i_mmap_sem);
4765 goto out_mutex;
4766 }
4767
Jan Kara01127842015-12-07 14:34:49 -05004768 ret = ext4_update_disksize_before_punch(inode, offset, len);
4769 if (ret) {
4770 up_write(&EXT4_I(inode)->i_mmap_sem);
Nikolay Borisov1d398342018-03-22 11:52:10 -04004771 goto out_mutex;
Jan Kara01127842015-12-07 14:34:49 -05004772 }
Jan Karaea3d7202015-12-07 14:28:03 -05004773 /* Now release the pages and zero block aligned part of pages */
4774 truncate_pagecache_range(inode, start, end - 1);
Deepa Dinamanieeca7ea2016-11-14 21:40:10 -05004775 inode->i_mtime = inode->i_ctime = current_time(inode);
Jan Karaea3d7202015-12-07 14:28:03 -05004776
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004777 ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size,
Tahsin Erdogan77a2e842017-08-05 22:15:45 -04004778 flags);
Jan Karaea3d7202015-12-07 14:28:03 -05004779 up_write(&EXT4_I(inode)->i_mmap_sem);
Lukas Czernerb8a86842014-03-18 18:05:35 -04004780 if (ret)
Nikolay Borisov1d398342018-03-22 11:52:10 -04004781 goto out_mutex;
Lukas Czernerb8a86842014-03-18 18:05:35 -04004782 }
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004783 if (!partial_begin && !partial_end)
Nikolay Borisov1d398342018-03-22 11:52:10 -04004784 goto out_mutex;
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004785
Dmitry Monakhov69dc9532014-08-27 18:33:49 -04004786 /*
4787 * In worst case we have to writeout two nonadjacent unwritten
4788 * blocks and update the inode
4789 */
4790 credits = (2 * ext4_ext_index_trans_blocks(inode, 2)) + 1;
4791 if (ext4_should_journal_data(inode))
4792 credits += 2;
4793 handle = ext4_journal_start(inode, EXT4_HT_MISC, credits);
Lukas Czernerb8a86842014-03-18 18:05:35 -04004794 if (IS_ERR(handle)) {
4795 ret = PTR_ERR(handle);
4796 ext4_std_error(inode->i_sb, ret);
Nikolay Borisov1d398342018-03-22 11:52:10 -04004797 goto out_mutex;
Lukas Czernerb8a86842014-03-18 18:05:35 -04004798 }
4799
Deepa Dinamanieeca7ea2016-11-14 21:40:10 -05004800 inode->i_mtime = inode->i_ctime = current_time(inode);
Lukas Czernere5b30412014-04-01 00:59:21 -04004801 if (new_size) {
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04004802 ext4_update_inode_size(inode, new_size);
Lukas Czernere5b30412014-04-01 00:59:21 -04004803 } else {
Lukas Czernerb8a86842014-03-18 18:05:35 -04004804 /*
4805 * Mark that we allocate beyond EOF so the subsequent truncate
4806 * can proceed even if the new size is the same as i_size.
4807 */
Eric Biggers9b02e492019-12-31 12:04:38 -06004808 if (offset + len > inode->i_size)
Lukas Czernerb8a86842014-03-18 18:05:35 -04004809 ext4_set_inode_flag(inode, EXT4_INODE_EOFBLOCKS);
4810 }
Lukas Czernerb8a86842014-03-18 18:05:35 -04004811 ext4_mark_inode_dirty(handle, inode);
4812
4813 /* Zero out partial block at the edges of the range */
4814 ret = ext4_zero_partial_blocks(handle, inode, offset, len);
Jan Kara67a7d5f2017-05-29 13:24:55 -04004815 if (ret >= 0)
4816 ext4_update_inode_fsync_trans(handle, inode, 1);
Lukas Czernerb8a86842014-03-18 18:05:35 -04004817
4818 if (file->f_flags & O_SYNC)
4819 ext4_handle_sync(handle);
4820
4821 ext4_journal_stop(handle);
Lukas Czernerb8a86842014-03-18 18:05:35 -04004822out_mutex:
Al Viro59551022016-01-22 15:40:57 -05004823 inode_unlock(inode);
Lukas Czernerb8a86842014-03-18 18:05:35 -04004824 return ret;
4825}
4826
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004827/*
4828 * preallocate space for a file. This implements ext4's fallocate file
4829 * operation, which gets called from sys_fallocate system call.
4830 * For block-mapped files, posix_fallocate should fall back to the method
4831 * of writing zeroes to the required new blocks (the same behavior which is
4832 * expected for file systems which do not support fallocate() system call).
4833 */
4834long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
4835{
4836 struct inode *inode = file_inode(file);
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004837 loff_t new_size = 0;
4838 unsigned int max_blocks;
4839 int ret = 0;
4840 int flags;
4841 ext4_lblk_t lblk;
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004842 unsigned int blkbits = inode->i_blkbits;
4843
Michael Halcrow2058f832015-04-12 00:55:10 -04004844 /*
4845 * Encrypted inodes can't handle collapse range or insert
4846 * range since we would need to re-encrypt blocks with a
4847 * different IV or XTS tweak (which are based on the logical
4848 * block number).
Michael Halcrow2058f832015-04-12 00:55:10 -04004849 */
Chandan Rajendra592ddec2018-12-12 15:20:10 +05304850 if (IS_ENCRYPTED(inode) &&
Eric Biggers457b1e32019-12-26 09:42:16 -06004851 (mode & (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE)))
Michael Halcrow2058f832015-04-12 00:55:10 -04004852 return -EOPNOTSUPP;
4853
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004854 /* Return error if mode is not supported */
4855 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
Namjae Jeon331573f2015-06-09 01:55:03 -04004856 FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE |
4857 FALLOC_FL_INSERT_RANGE))
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004858 return -EOPNOTSUPP;
4859
4860 if (mode & FALLOC_FL_PUNCH_HOLE)
4861 return ext4_punch_hole(inode, offset, len);
4862
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004863 ret = ext4_convert_inline_data(inode);
4864 if (ret)
4865 return ret;
4866
Theodore Ts'o40c406c2014-04-12 22:53:53 -04004867 if (mode & FALLOC_FL_COLLAPSE_RANGE)
4868 return ext4_collapse_range(inode, offset, len);
4869
Namjae Jeon331573f2015-06-09 01:55:03 -04004870 if (mode & FALLOC_FL_INSERT_RANGE)
4871 return ext4_insert_range(inode, offset, len);
4872
Lukas Czernerb8a86842014-03-18 18:05:35 -04004873 if (mode & FALLOC_FL_ZERO_RANGE)
4874 return ext4_zero_range(file, offset, len, mode);
4875
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004876 trace_ext4_fallocate_enter(inode, offset, len, mode);
4877 lblk = offset >> blkbits;
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004878
Fabian Frederick518eaa62016-09-15 11:55:01 -04004879 max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);
Lukas Czerner556615d2014-04-20 23:45:47 -04004880 flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004881 if (mode & FALLOC_FL_KEEP_SIZE)
4882 flags |= EXT4_GET_BLOCKS_KEEP_SIZE;
4883
Al Viro59551022016-01-22 15:40:57 -05004884 inode_lock(inode);
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004885
Davide Italiano280227a2015-05-02 23:21:15 -04004886 /*
4887 * We only support preallocation for extent-based files only
4888 */
4889 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
4890 ret = -EOPNOTSUPP;
4891 goto out;
4892 }
4893
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004894 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
Eric Biggers9b02e492019-12-31 12:04:38 -06004895 (offset + len > inode->i_size ||
Theodore Ts'o51e3ae82017-10-06 23:09:55 -04004896 offset + len > EXT4_I(inode)->i_disksize)) {
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004897 new_size = offset + len;
4898 ret = inode_newsize_ok(inode, new_size);
4899 if (ret)
4900 goto out;
4901 }
4902
Jan Kara17048e82015-12-07 14:29:17 -05004903 /* Wait all existing dio workers, newcomers will block on i_mutex */
Jan Kara17048e82015-12-07 14:29:17 -05004904 inode_dio_wait(inode);
4905
Tahsin Erdogan77a2e842017-08-05 22:15:45 -04004906 ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size, flags);
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004907 if (ret)
4908 goto out;
4909
Dmitry Monakhovc174e6d2014-08-27 18:40:00 -04004910 if (file->f_flags & O_SYNC && EXT4_SB(inode->i_sb)->s_journal) {
4911 ret = jbd2_complete_transaction(EXT4_SB(inode->i_sb)->s_journal,
4912 EXT4_I(inode)->i_sync_tid);
Lukas Czernerf282ac12014-03-18 17:44:35 -04004913 }
Lukas Czernerf282ac12014-03-18 17:44:35 -04004914out:
Al Viro59551022016-01-22 15:40:57 -05004915 inode_unlock(inode);
Lukas Czerner0e8b6872014-03-18 18:03:51 -04004916 trace_ext4_fallocate_exit(inode, offset, max_blocks, ret);
4917 return ret;
Amit Aroraa2df2a62007-07-17 21:42:41 -04004918}
Eric Sandeen6873fa02008-10-07 00:46:36 -04004919
4920/*
Mingming Cao00314622009-09-28 15:49:08 -04004921 * This function convert a range of blocks to written extents
4922 * The caller of this function will pass the start offset and the size.
4923 * all unwritten extents within this range will be converted to
4924 * written extents.
4925 *
4926 * This function is called from the direct IO end io call back
4927 * function, to convert the fallocated extents after IO is completed.
Mingming109f5562009-11-10 10:48:08 -05004928 * Returns 0 on success.
Mingming Cao00314622009-09-28 15:49:08 -04004929 */
Jan Kara6b523df2013-06-04 13:21:11 -04004930int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
4931 loff_t offset, ssize_t len)
Mingming Cao00314622009-09-28 15:49:08 -04004932{
Mingming Cao00314622009-09-28 15:49:08 -04004933 unsigned int max_blocks;
4934 int ret = 0;
4935 int ret2 = 0;
Theodore Ts'o2ed88682010-05-16 20:00:00 -04004936 struct ext4_map_blocks map;
Ritesh Harjania00713e2019-10-16 13:07:08 +05304937 unsigned int blkbits = inode->i_blkbits;
4938 unsigned int credits = 0;
Mingming Cao00314622009-09-28 15:49:08 -04004939
Theodore Ts'o2ed88682010-05-16 20:00:00 -04004940 map.m_lblk = offset >> blkbits;
Fabian Frederick518eaa62016-09-15 11:55:01 -04004941 max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);
4942
Ritesh Harjania00713e2019-10-16 13:07:08 +05304943 if (!handle) {
Jan Kara6b523df2013-06-04 13:21:11 -04004944 /*
4945 * credits to insert 1 extent into extent tree
4946 */
4947 credits = ext4_chunk_trans_blocks(inode, max_blocks);
4948 }
Mingming Cao00314622009-09-28 15:49:08 -04004949 while (ret >= 0 && ret < max_blocks) {
Theodore Ts'o2ed88682010-05-16 20:00:00 -04004950 map.m_lblk += ret;
4951 map.m_len = (max_blocks -= ret);
Jan Kara6b523df2013-06-04 13:21:11 -04004952 if (credits) {
4953 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
4954 credits);
4955 if (IS_ERR(handle)) {
4956 ret = PTR_ERR(handle);
4957 break;
4958 }
Mingming Cao00314622009-09-28 15:49:08 -04004959 }
Theodore Ts'o2ed88682010-05-16 20:00:00 -04004960 ret = ext4_map_blocks(handle, inode, &map,
Jiaying Zhangc7064ef2010-03-02 13:28:44 -05004961 EXT4_GET_BLOCKS_IO_CONVERT_EXT);
Lukas Czernerb06acd32013-01-28 21:21:12 -05004962 if (ret <= 0)
4963 ext4_warning(inode->i_sb,
4964 "inode #%lu: block %u: len %u: "
4965 "ext4_ext_map_blocks returned %d",
4966 inode->i_ino, map.m_lblk,
4967 map.m_len, ret);
Mingming Cao00314622009-09-28 15:49:08 -04004968 ext4_mark_inode_dirty(handle, inode);
Jan Kara6b523df2013-06-04 13:21:11 -04004969 if (credits)
4970 ret2 = ext4_journal_stop(handle);
4971 if (ret <= 0 || ret2)
Mingming Cao00314622009-09-28 15:49:08 -04004972 break;
4973 }
4974 return ret > 0 ? ret2 : ret;
4975}
Yongqiang Yang6d9c85e2011-02-27 17:25:47 -05004976
Ritesh Harjania00713e2019-10-16 13:07:08 +05304977int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end)
4978{
4979 int ret, err = 0;
Ritesh Harjanic8cc8812019-10-16 13:07:10 +05304980 struct ext4_io_end_vec *io_end_vec;
Ritesh Harjania00713e2019-10-16 13:07:08 +05304981
4982 /*
4983 * This is somewhat ugly but the idea is clear: When transaction is
4984 * reserved, everything goes into it. Otherwise we rather start several
4985 * smaller transactions for conversion of each extent separately.
4986 */
4987 if (handle) {
4988 handle = ext4_journal_start_reserved(handle,
4989 EXT4_HT_EXT_CONVERT);
4990 if (IS_ERR(handle))
4991 return PTR_ERR(handle);
4992 }
4993
Ritesh Harjanic8cc8812019-10-16 13:07:10 +05304994 list_for_each_entry(io_end_vec, &io_end->list_vec, list) {
4995 ret = ext4_convert_unwritten_extents(handle, io_end->inode,
4996 io_end_vec->offset,
4997 io_end_vec->size);
4998 if (ret)
4999 break;
5000 }
5001
Ritesh Harjania00713e2019-10-16 13:07:08 +05305002 if (handle)
5003 err = ext4_journal_stop(handle);
5004
5005 return ret < 0 ? ret : err;
5006}
5007
Mingming Cao00314622009-09-28 15:49:08 -04005008/*
Zheng Liu69eb33d2013-02-18 00:31:07 -05005009 * If newes is not existing extent (newes->ec_pblk equals zero) find
5010 * delayed extent at start of newes and update newes accordingly and
Lukas Czerner91dd8c12012-11-28 12:32:26 -05005011 * return start of the next delayed extent.
5012 *
Zheng Liu69eb33d2013-02-18 00:31:07 -05005013 * If newes is existing extent (newes->ec_pblk is not equal zero)
Lukas Czerner91dd8c12012-11-28 12:32:26 -05005014 * return start of next delayed extent or EXT_MAX_BLOCKS if no delayed
Zheng Liu69eb33d2013-02-18 00:31:07 -05005015 * extent found. Leave newes unmodified.
Eric Sandeen6873fa02008-10-07 00:46:36 -04005016 */
Lukas Czerner91dd8c12012-11-28 12:32:26 -05005017static int ext4_find_delayed_extent(struct inode *inode,
Zheng Liu69eb33d2013-02-18 00:31:07 -05005018 struct extent_status *newes)
Eric Sandeen6873fa02008-10-07 00:46:36 -04005019{
Zheng Liub3aff3e2012-11-08 21:57:37 -05005020 struct extent_status es;
Zheng Liube401362013-02-18 00:27:26 -05005021 ext4_lblk_t block, next_del;
Eric Sandeen6873fa02008-10-07 00:46:36 -04005022
Zheng Liu69eb33d2013-02-18 00:31:07 -05005023 if (newes->es_pblk == 0) {
Eric Whitneyad431022018-10-01 14:10:39 -04005024 ext4_es_find_extent_range(inode, &ext4_es_is_delayed,
5025 newes->es_lblk,
5026 newes->es_lblk + newes->es_len - 1,
5027 &es);
Yan, Zhenge30b5dc2013-05-03 02:15:52 -04005028
Yongqiang Yang6d9c85e2011-02-27 17:25:47 -05005029 /*
Zheng Liu69eb33d2013-02-18 00:31:07 -05005030 * No extent in extent-tree contains block @newes->es_pblk,
Yongqiang Yang6d9c85e2011-02-27 17:25:47 -05005031 * then the block may stay in 1)a hole or 2)delayed-extent.
Yongqiang Yang6d9c85e2011-02-27 17:25:47 -05005032 */
Zheng Liu06b0c882013-02-18 00:26:51 -05005033 if (es.es_len == 0)
Zheng Liub3aff3e2012-11-08 21:57:37 -05005034 /* A hole found. */
Lukas Czerner91dd8c12012-11-28 12:32:26 -05005035 return 0;
Yongqiang Yang6d9c85e2011-02-27 17:25:47 -05005036
Zheng Liu69eb33d2013-02-18 00:31:07 -05005037 if (es.es_lblk > newes->es_lblk) {
Zheng Liub3aff3e2012-11-08 21:57:37 -05005038 /* A hole found. */
Zheng Liu69eb33d2013-02-18 00:31:07 -05005039 newes->es_len = min(es.es_lblk - newes->es_lblk,
5040 newes->es_len);
Lukas Czerner91dd8c12012-11-28 12:32:26 -05005041 return 0;
Eric Sandeen6873fa02008-10-07 00:46:36 -04005042 }
Yongqiang Yang6d9c85e2011-02-27 17:25:47 -05005043
Zheng Liu69eb33d2013-02-18 00:31:07 -05005044 newes->es_len = es.es_lblk + es.es_len - newes->es_lblk;
Eric Sandeen6873fa02008-10-07 00:46:36 -04005045 }
5046
Zheng Liu69eb33d2013-02-18 00:31:07 -05005047 block = newes->es_lblk + newes->es_len;
Eric Whitneyad431022018-10-01 14:10:39 -04005048 ext4_es_find_extent_range(inode, &ext4_es_is_delayed, block,
5049 EXT_MAX_BLOCKS, &es);
Zheng Liube401362013-02-18 00:27:26 -05005050 if (es.es_len == 0)
5051 next_del = EXT_MAX_BLOCKS;
5052 else
5053 next_del = es.es_lblk;
5054
Lukas Czerner91dd8c12012-11-28 12:32:26 -05005055 return next_del;
Eric Sandeen6873fa02008-10-07 00:46:36 -04005056}
Eric Sandeen6873fa02008-10-07 00:46:36 -04005057
Aneesh Kumar K.V3a06d772008-11-22 15:04:59 -05005058static int ext4_xattr_fiemap(struct inode *inode,
5059 struct fiemap_extent_info *fieinfo)
Eric Sandeen6873fa02008-10-07 00:46:36 -04005060{
5061 __u64 physical = 0;
5062 __u64 length;
5063 __u32 flags = FIEMAP_EXTENT_LAST;
5064 int blockbits = inode->i_sb->s_blocksize_bits;
5065 int error = 0;
5066
5067 /* in-inode? */
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05005068 if (ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {
Eric Sandeen6873fa02008-10-07 00:46:36 -04005069 struct ext4_iloc iloc;
5070 int offset; /* offset of xattr in inode */
5071
5072 error = ext4_get_inode_loc(inode, &iloc);
5073 if (error)
5074 return error;
Jan Karaa60697f2013-05-31 19:38:56 -04005075 physical = (__u64)iloc.bh->b_blocknr << blockbits;
Eric Sandeen6873fa02008-10-07 00:46:36 -04005076 offset = EXT4_GOOD_OLD_INODE_SIZE +
5077 EXT4_I(inode)->i_extra_isize;
5078 physical += offset;
5079 length = EXT4_SB(inode->i_sb)->s_inode_size - offset;
5080 flags |= FIEMAP_EXTENT_DATA_INLINE;
Curt Wohlgemuthfd2dd9f2010-04-03 17:44:16 -04005081 brelse(iloc.bh);
Eric Sandeen6873fa02008-10-07 00:46:36 -04005082 } else { /* external block */
Jan Karaa60697f2013-05-31 19:38:56 -04005083 physical = (__u64)EXT4_I(inode)->i_file_acl << blockbits;
Eric Sandeen6873fa02008-10-07 00:46:36 -04005084 length = inode->i_sb->s_blocksize;
5085 }
5086
5087 if (physical)
5088 error = fiemap_fill_next_extent(fieinfo, 0, physical,
5089 length, flags);
5090 return (error < 0 ? error : 0);
5091}
5092
Theodore Ts'obb5835e2019-08-11 16:32:41 -04005093static int _ext4_fiemap(struct inode *inode,
5094 struct fiemap_extent_info *fieinfo,
5095 __u64 start, __u64 len,
5096 int (*fill)(struct inode *, ext4_lblk_t,
5097 ext4_lblk_t,
5098 struct fiemap_extent_info *))
Eric Sandeen6873fa02008-10-07 00:46:36 -04005099{
5100 ext4_lblk_t start_blk;
Theodore Ts'obb5835e2019-08-11 16:32:41 -04005101 u32 ext4_fiemap_flags = FIEMAP_FLAG_SYNC|FIEMAP_FLAG_XATTR;
5102
Eric Sandeen6873fa02008-10-07 00:46:36 -04005103 int error = 0;
5104
Tao Ma94191982012-12-10 14:06:02 -05005105 if (ext4_has_inline_data(inode)) {
5106 int has_inline = 1;
5107
Dmitry Monakhovd952d692014-12-02 16:11:20 -05005108 error = ext4_inline_data_fiemap(inode, fieinfo, &has_inline,
5109 start, len);
Tao Ma94191982012-12-10 14:06:02 -05005110
5111 if (has_inline)
5112 return error;
5113 }
5114
Theodore Ts'o7869a4a2013-08-16 22:05:14 -04005115 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
5116 error = ext4_ext_precache(inode);
5117 if (error)
5118 return error;
Theodore Ts'obb5835e2019-08-11 16:32:41 -04005119 fieinfo->fi_flags &= ~FIEMAP_FLAG_CACHE;
Theodore Ts'o7869a4a2013-08-16 22:05:14 -04005120 }
5121
Eric Sandeen6873fa02008-10-07 00:46:36 -04005122 /* fallback to generic here if not in extents fmt */
Theodore Ts'obb5835e2019-08-11 16:32:41 -04005123 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) &&
5124 fill == ext4_fill_fiemap_extents)
Theodore Ts'oad7fefb2015-01-02 15:16:00 -05005125 return generic_block_fiemap(inode, fieinfo, start, len,
5126 ext4_get_block);
Eric Sandeen6873fa02008-10-07 00:46:36 -04005127
Theodore Ts'obb5835e2019-08-11 16:32:41 -04005128 if (fill == ext4_fill_es_cache_info)
5129 ext4_fiemap_flags &= FIEMAP_FLAG_XATTR;
5130 if (fiemap_check_flags(fieinfo, ext4_fiemap_flags))
Eric Sandeen6873fa02008-10-07 00:46:36 -04005131 return -EBADR;
5132
5133 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
5134 error = ext4_xattr_fiemap(inode, fieinfo);
5135 } else {
Leonard Michlmayraca92ff2010-03-04 17:07:28 -05005136 ext4_lblk_t len_blks;
5137 __u64 last_blk;
5138
Eric Sandeen6873fa02008-10-07 00:46:36 -04005139 start_blk = start >> inode->i_sb->s_blocksize_bits;
Leonard Michlmayraca92ff2010-03-04 17:07:28 -05005140 last_blk = (start + len - 1) >> inode->i_sb->s_blocksize_bits;
Lukas Czernerf17722f2011-06-06 00:05:17 -04005141 if (last_blk >= EXT_MAX_BLOCKS)
5142 last_blk = EXT_MAX_BLOCKS-1;
Leonard Michlmayraca92ff2010-03-04 17:07:28 -05005143 len_blks = ((ext4_lblk_t) last_blk) - start_blk + 1;
Eric Sandeen6873fa02008-10-07 00:46:36 -04005144
5145 /*
Lukas Czerner91dd8c12012-11-28 12:32:26 -05005146 * Walk the extent tree gathering extent information
5147 * and pushing extents back to the user.
Eric Sandeen6873fa02008-10-07 00:46:36 -04005148 */
Theodore Ts'obb5835e2019-08-11 16:32:41 -04005149 error = fill(inode, start_blk, len_blks, fieinfo);
Eric Sandeen6873fa02008-10-07 00:46:36 -04005150 }
Eric Sandeen6873fa02008-10-07 00:46:36 -04005151 return error;
5152}
Namjae Jeon9eb79482014-02-23 15:18:59 -05005153
Theodore Ts'obb5835e2019-08-11 16:32:41 -04005154int ext4_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
5155 __u64 start, __u64 len)
5156{
5157 return _ext4_fiemap(inode, fieinfo, start, len,
5158 ext4_fill_fiemap_extents);
5159}
5160
5161int ext4_get_es_cache(struct inode *inode, struct fiemap_extent_info *fieinfo,
5162 __u64 start, __u64 len)
5163{
5164 if (ext4_has_inline_data(inode)) {
5165 int has_inline;
5166
5167 down_read(&EXT4_I(inode)->xattr_sem);
5168 has_inline = ext4_has_inline_data(inode);
5169 up_read(&EXT4_I(inode)->xattr_sem);
5170 if (has_inline)
5171 return 0;
5172 }
5173
5174 return _ext4_fiemap(inode, fieinfo, start, len,
5175 ext4_fill_es_cache_info);
5176}
5177
5178
Namjae Jeon9eb79482014-02-23 15:18:59 -05005179/*
5180 * ext4_access_path:
5181 * Function to access the path buffer for marking it dirty.
5182 * It also checks if there are sufficient credits left in the journal handle
5183 * to update path.
5184 */
5185static int
5186ext4_access_path(handle_t *handle, struct inode *inode,
5187 struct ext4_ext_path *path)
5188{
5189 int credits, err;
5190
5191 if (!ext4_handle_valid(handle))
5192 return 0;
5193
5194 /*
5195 * Check if need to extend journal credits
5196 * 3 for leaf, sb, and inode plus 2 (bmap and group
5197 * descriptor) for each block group; assume two block
5198 * groups
5199 */
Jan Karaa4130362019-11-05 17:44:16 +01005200 credits = ext4_writepage_trans_blocks(inode);
Jan Kara83448bd2019-11-05 17:44:29 +01005201 err = ext4_datasem_ensure_credits(handle, inode, 7, credits, 0);
Jan Karaa4130362019-11-05 17:44:16 +01005202 if (err < 0)
5203 return err;
Namjae Jeon9eb79482014-02-23 15:18:59 -05005204
5205 err = ext4_ext_get_access(handle, inode, path);
5206 return err;
5207}
5208
5209/*
5210 * ext4_ext_shift_path_extents:
5211 * Shift the extents of a path structure lying between path[depth].p_ext
Namjae Jeon331573f2015-06-09 01:55:03 -04005212 * and EXT_LAST_EXTENT(path[depth].p_hdr), by @shift blocks. @SHIFT tells
5213 * if it is right shift or left shift operation.
Namjae Jeon9eb79482014-02-23 15:18:59 -05005214 */
5215static int
5216ext4_ext_shift_path_extents(struct ext4_ext_path *path, ext4_lblk_t shift,
5217 struct inode *inode, handle_t *handle,
Namjae Jeon331573f2015-06-09 01:55:03 -04005218 enum SHIFT_DIRECTION SHIFT)
Namjae Jeon9eb79482014-02-23 15:18:59 -05005219{
5220 int depth, err = 0;
5221 struct ext4_extent *ex_start, *ex_last;
zhengbin4756ee12019-12-25 10:45:59 +08005222 bool update = false;
Namjae Jeon9eb79482014-02-23 15:18:59 -05005223 depth = path->p_depth;
5224
5225 while (depth >= 0) {
5226 if (depth == path->p_depth) {
5227 ex_start = path[depth].p_ext;
5228 if (!ex_start)
Darrick J. Wong6a797d22015-10-17 16:16:04 -04005229 return -EFSCORRUPTED;
Namjae Jeon9eb79482014-02-23 15:18:59 -05005230
5231 ex_last = EXT_LAST_EXTENT(path[depth].p_hdr);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005232
5233 err = ext4_access_path(handle, inode, path + depth);
5234 if (err)
5235 goto out;
5236
5237 if (ex_start == EXT_FIRST_EXTENT(path[depth].p_hdr))
zhengbin4756ee12019-12-25 10:45:59 +08005238 update = true;
Namjae Jeon9eb79482014-02-23 15:18:59 -05005239
Namjae Jeon9eb79482014-02-23 15:18:59 -05005240 while (ex_start <= ex_last) {
Namjae Jeon331573f2015-06-09 01:55:03 -04005241 if (SHIFT == SHIFT_LEFT) {
5242 le32_add_cpu(&ex_start->ee_block,
5243 -shift);
5244 /* Try to merge to the left. */
5245 if ((ex_start >
5246 EXT_FIRST_EXTENT(path[depth].p_hdr))
5247 &&
5248 ext4_ext_try_to_merge_right(inode,
5249 path, ex_start - 1))
5250 ex_last--;
5251 else
5252 ex_start++;
5253 } else {
5254 le32_add_cpu(&ex_last->ee_block, shift);
5255 ext4_ext_try_to_merge_right(inode, path,
5256 ex_last);
Lukas Czerner6dd834e2014-04-18 10:55:24 -04005257 ex_last--;
Namjae Jeon331573f2015-06-09 01:55:03 -04005258 }
Namjae Jeon9eb79482014-02-23 15:18:59 -05005259 }
5260 err = ext4_ext_dirty(handle, inode, path + depth);
5261 if (err)
5262 goto out;
5263
5264 if (--depth < 0 || !update)
5265 break;
5266 }
5267
5268 /* Update index too */
5269 err = ext4_access_path(handle, inode, path + depth);
5270 if (err)
5271 goto out;
5272
Namjae Jeon331573f2015-06-09 01:55:03 -04005273 if (SHIFT == SHIFT_LEFT)
5274 le32_add_cpu(&path[depth].p_idx->ei_block, -shift);
5275 else
5276 le32_add_cpu(&path[depth].p_idx->ei_block, shift);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005277 err = ext4_ext_dirty(handle, inode, path + depth);
5278 if (err)
5279 goto out;
5280
5281 /* we are done if current index is not a starting index */
5282 if (path[depth].p_idx != EXT_FIRST_INDEX(path[depth].p_hdr))
5283 break;
5284
5285 depth--;
5286 }
5287
5288out:
5289 return err;
5290}
5291
5292/*
5293 * ext4_ext_shift_extents:
Namjae Jeon331573f2015-06-09 01:55:03 -04005294 * All the extents which lies in the range from @start to the last allocated
5295 * block for the @inode are shifted either towards left or right (depending
5296 * upon @SHIFT) by @shift blocks.
Namjae Jeon9eb79482014-02-23 15:18:59 -05005297 * On success, 0 is returned, error otherwise.
5298 */
5299static int
5300ext4_ext_shift_extents(struct inode *inode, handle_t *handle,
Namjae Jeon331573f2015-06-09 01:55:03 -04005301 ext4_lblk_t start, ext4_lblk_t shift,
5302 enum SHIFT_DIRECTION SHIFT)
Namjae Jeon9eb79482014-02-23 15:18:59 -05005303{
5304 struct ext4_ext_path *path;
5305 int ret = 0, depth;
5306 struct ext4_extent *extent;
Namjae Jeon331573f2015-06-09 01:55:03 -04005307 ext4_lblk_t stop, *iterator, ex_start, ex_end;
Namjae Jeon9eb79482014-02-23 15:18:59 -05005308
5309 /* Let path point to the last extent */
Roman Pen03e916f2017-01-08 21:00:35 -05005310 path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL,
5311 EXT4_EX_NOCACHE);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005312 if (IS_ERR(path))
5313 return PTR_ERR(path);
5314
5315 depth = path->p_depth;
5316 extent = path[depth].p_ext;
Theodore Ts'oee4bd0d92014-09-01 14:41:09 -04005317 if (!extent)
5318 goto out;
Namjae Jeon9eb79482014-02-23 15:18:59 -05005319
Roman Pen2a9b8cb2017-01-08 20:59:35 -05005320 stop = le32_to_cpu(extent->ee_block);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005321
Namjae Jeon331573f2015-06-09 01:55:03 -04005322 /*
Eric Biggers349fa7d2018-04-12 11:48:09 -04005323 * For left shifts, make sure the hole on the left is big enough to
5324 * accommodate the shift. For right shifts, make sure the last extent
5325 * won't be shifted beyond EXT_MAX_BLOCKS.
Namjae Jeon331573f2015-06-09 01:55:03 -04005326 */
5327 if (SHIFT == SHIFT_LEFT) {
Roman Pen03e916f2017-01-08 21:00:35 -05005328 path = ext4_find_extent(inode, start - 1, &path,
5329 EXT4_EX_NOCACHE);
Namjae Jeon331573f2015-06-09 01:55:03 -04005330 if (IS_ERR(path))
5331 return PTR_ERR(path);
5332 depth = path->p_depth;
5333 extent = path[depth].p_ext;
5334 if (extent) {
5335 ex_start = le32_to_cpu(extent->ee_block);
5336 ex_end = le32_to_cpu(extent->ee_block) +
5337 ext4_ext_get_actual_len(extent);
5338 } else {
5339 ex_start = 0;
5340 ex_end = 0;
5341 }
Namjae Jeon9eb79482014-02-23 15:18:59 -05005342
Namjae Jeon331573f2015-06-09 01:55:03 -04005343 if ((start == ex_start && shift > ex_start) ||
5344 (shift > start - ex_end)) {
Eric Biggers349fa7d2018-04-12 11:48:09 -04005345 ret = -EINVAL;
5346 goto out;
5347 }
5348 } else {
5349 if (shift > EXT_MAX_BLOCKS -
5350 (stop + ext4_ext_get_actual_len(extent))) {
5351 ret = -EINVAL;
5352 goto out;
Namjae Jeon331573f2015-06-09 01:55:03 -04005353 }
Dmitry Monakhov8dc79ec2014-04-13 15:05:42 -04005354 }
Namjae Jeon9eb79482014-02-23 15:18:59 -05005355
Namjae Jeon331573f2015-06-09 01:55:03 -04005356 /*
5357 * In case of left shift, iterator points to start and it is increased
5358 * till we reach stop. In case of right shift, iterator points to stop
5359 * and it is decreased till we reach start.
5360 */
5361 if (SHIFT == SHIFT_LEFT)
5362 iterator = &start;
5363 else
5364 iterator = &stop;
Namjae Jeon9eb79482014-02-23 15:18:59 -05005365
Roman Pen2a9b8cb2017-01-08 20:59:35 -05005366 /*
5367 * Its safe to start updating extents. Start and stop are unsigned, so
5368 * in case of right shift if extent with 0 block is reached, iterator
5369 * becomes NULL to indicate the end of the loop.
5370 */
5371 while (iterator && start <= stop) {
Roman Pen03e916f2017-01-08 21:00:35 -05005372 path = ext4_find_extent(inode, *iterator, &path,
5373 EXT4_EX_NOCACHE);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005374 if (IS_ERR(path))
5375 return PTR_ERR(path);
5376 depth = path->p_depth;
5377 extent = path[depth].p_ext;
Dmitry Monakhova18ed352014-04-13 15:41:13 -04005378 if (!extent) {
5379 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
Namjae Jeon331573f2015-06-09 01:55:03 -04005380 (unsigned long) *iterator);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04005381 return -EFSCORRUPTED;
Dmitry Monakhova18ed352014-04-13 15:41:13 -04005382 }
Namjae Jeon331573f2015-06-09 01:55:03 -04005383 if (SHIFT == SHIFT_LEFT && *iterator >
5384 le32_to_cpu(extent->ee_block)) {
Namjae Jeon9eb79482014-02-23 15:18:59 -05005385 /* Hole, move to the next extent */
Dmitry Monakhovf8fb4f42014-08-30 23:50:56 -04005386 if (extent < EXT_LAST_EXTENT(path[depth].p_hdr)) {
5387 path[depth].p_ext++;
5388 } else {
Namjae Jeon331573f2015-06-09 01:55:03 -04005389 *iterator = ext4_ext_next_allocated_block(path);
Dmitry Monakhovf8fb4f42014-08-30 23:50:56 -04005390 continue;
Namjae Jeon9eb79482014-02-23 15:18:59 -05005391 }
5392 }
Namjae Jeon331573f2015-06-09 01:55:03 -04005393
5394 if (SHIFT == SHIFT_LEFT) {
5395 extent = EXT_LAST_EXTENT(path[depth].p_hdr);
5396 *iterator = le32_to_cpu(extent->ee_block) +
5397 ext4_ext_get_actual_len(extent);
5398 } else {
5399 extent = EXT_FIRST_EXTENT(path[depth].p_hdr);
Roman Pen2a9b8cb2017-01-08 20:59:35 -05005400 if (le32_to_cpu(extent->ee_block) > 0)
5401 *iterator = le32_to_cpu(extent->ee_block) - 1;
5402 else
5403 /* Beginning is reached, end of the loop */
5404 iterator = NULL;
Namjae Jeon331573f2015-06-09 01:55:03 -04005405 /* Update path extent in case we need to stop */
5406 while (le32_to_cpu(extent->ee_block) < start)
5407 extent++;
5408 path[depth].p_ext = extent;
5409 }
Namjae Jeon9eb79482014-02-23 15:18:59 -05005410 ret = ext4_ext_shift_path_extents(path, shift, inode,
Namjae Jeon331573f2015-06-09 01:55:03 -04005411 handle, SHIFT);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005412 if (ret)
5413 break;
5414 }
Theodore Ts'oee4bd0d92014-09-01 14:41:09 -04005415out:
5416 ext4_ext_drop_refs(path);
5417 kfree(path);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005418 return ret;
5419}
5420
5421/*
5422 * ext4_collapse_range:
5423 * This implements the fallocate's collapse range functionality for ext4
5424 * Returns: 0 and non-zero on error.
5425 */
Eric Biggers43f81672019-12-31 12:04:40 -06005426static int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len)
Namjae Jeon9eb79482014-02-23 15:18:59 -05005427{
5428 struct super_block *sb = inode->i_sb;
5429 ext4_lblk_t punch_start, punch_stop;
5430 handle_t *handle;
5431 unsigned int credits;
Namjae Jeona8680e02014-04-19 16:37:31 -04005432 loff_t new_size, ioffset;
Namjae Jeon9eb79482014-02-23 15:18:59 -05005433 int ret;
5434
Theodore Ts'ob9576fc2015-05-15 00:24:10 -04005435 /*
5436 * We need to test this early because xfstests assumes that a
5437 * collapse range of (0, 1) will return EOPNOTSUPP if the file
5438 * system does not support collapse range.
5439 */
5440 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5441 return -EOPNOTSUPP;
5442
Eric Biggers9b02e492019-12-31 12:04:38 -06005443 /* Collapse range works only on fs cluster size aligned regions. */
5444 if (!IS_ALIGNED(offset | len, EXT4_CLUSTER_SIZE(sb)))
Namjae Jeon9eb79482014-02-23 15:18:59 -05005445 return -EINVAL;
5446
Namjae Jeon9eb79482014-02-23 15:18:59 -05005447 trace_ext4_collapse_range(inode, offset, len);
5448
5449 punch_start = offset >> EXT4_BLOCK_SIZE_BITS(sb);
5450 punch_stop = (offset + len) >> EXT4_BLOCK_SIZE_BITS(sb);
5451
Namjae Jeon1ce01c42014-04-10 22:58:20 -04005452 /* Call ext4_force_commit to flush all data in case of data=journal. */
5453 if (ext4_should_journal_data(inode)) {
5454 ret = ext4_force_commit(inode->i_sb);
5455 if (ret)
5456 return ret;
5457 }
5458
Al Viro59551022016-01-22 15:40:57 -05005459 inode_lock(inode);
Lukas Czerner23fffa92014-04-12 09:56:41 -04005460 /*
5461 * There is no need to overlap collapse range with EOF, in which case
5462 * it is effectively a truncate operation
5463 */
Eric Biggers9b02e492019-12-31 12:04:38 -06005464 if (offset + len >= inode->i_size) {
Lukas Czerner23fffa92014-04-12 09:56:41 -04005465 ret = -EINVAL;
5466 goto out_mutex;
5467 }
5468
Namjae Jeon9eb79482014-02-23 15:18:59 -05005469 /* Currently just for extent based files */
5470 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
5471 ret = -EOPNOTSUPP;
5472 goto out_mutex;
5473 }
5474
Namjae Jeon9eb79482014-02-23 15:18:59 -05005475 /* Wait for existing dio to complete */
Namjae Jeon9eb79482014-02-23 15:18:59 -05005476 inode_dio_wait(inode);
5477
Jan Karaea3d7202015-12-07 14:28:03 -05005478 /*
5479 * Prevent page faults from reinstantiating pages we have released from
5480 * page cache.
5481 */
5482 down_write(&EXT4_I(inode)->i_mmap_sem);
Ross Zwisler430657b2018-07-29 17:00:22 -04005483
5484 ret = ext4_break_layouts(inode);
5485 if (ret)
5486 goto out_mmap;
5487
Jan Kara32ebffd2015-12-07 14:31:11 -05005488 /*
5489 * Need to round down offset to be aligned with page size boundary
5490 * for page size > block size.
5491 */
5492 ioffset = round_down(offset, PAGE_SIZE);
5493 /*
5494 * Write tail of the last page before removed range since it will get
5495 * removed from the page cache below.
5496 */
5497 ret = filemap_write_and_wait_range(inode->i_mapping, ioffset, offset);
5498 if (ret)
5499 goto out_mmap;
5500 /*
5501 * Write data that will be shifted to preserve them when discarding
5502 * page cache below. We are also protected from pages becoming dirty
5503 * by i_mmap_sem.
5504 */
5505 ret = filemap_write_and_wait_range(inode->i_mapping, offset + len,
5506 LLONG_MAX);
5507 if (ret)
5508 goto out_mmap;
Jan Karaea3d7202015-12-07 14:28:03 -05005509 truncate_pagecache(inode, ioffset);
5510
Namjae Jeon9eb79482014-02-23 15:18:59 -05005511 credits = ext4_writepage_trans_blocks(inode);
5512 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
5513 if (IS_ERR(handle)) {
5514 ret = PTR_ERR(handle);
Jan Karaea3d7202015-12-07 14:28:03 -05005515 goto out_mmap;
Namjae Jeon9eb79482014-02-23 15:18:59 -05005516 }
5517
5518 down_write(&EXT4_I(inode)->i_data_sem);
5519 ext4_discard_preallocations(inode);
5520
5521 ret = ext4_es_remove_extent(inode, punch_start,
Lukas Czerner2c1d2322014-04-18 10:43:21 -04005522 EXT_MAX_BLOCKS - punch_start);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005523 if (ret) {
5524 up_write(&EXT4_I(inode)->i_data_sem);
5525 goto out_stop;
5526 }
5527
5528 ret = ext4_ext_remove_space(inode, punch_start, punch_stop - 1);
5529 if (ret) {
5530 up_write(&EXT4_I(inode)->i_data_sem);
5531 goto out_stop;
5532 }
Lukas Czerneref24f6c2014-04-18 10:50:23 -04005533 ext4_discard_preallocations(inode);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005534
5535 ret = ext4_ext_shift_extents(inode, handle, punch_stop,
Namjae Jeon331573f2015-06-09 01:55:03 -04005536 punch_stop - punch_start, SHIFT_LEFT);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005537 if (ret) {
5538 up_write(&EXT4_I(inode)->i_data_sem);
5539 goto out_stop;
5540 }
5541
Eric Biggers9b02e492019-12-31 12:04:38 -06005542 new_size = inode->i_size - len;
Lukas Czerner9337d5d2014-04-18 10:48:25 -04005543 i_size_write(inode, new_size);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005544 EXT4_I(inode)->i_disksize = new_size;
5545
Namjae Jeon9eb79482014-02-23 15:18:59 -05005546 up_write(&EXT4_I(inode)->i_data_sem);
5547 if (IS_SYNC(inode))
5548 ext4_handle_sync(handle);
Deepa Dinamanieeca7ea2016-11-14 21:40:10 -05005549 inode->i_mtime = inode->i_ctime = current_time(inode);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005550 ext4_mark_inode_dirty(handle, inode);
Jan Kara67a7d5f2017-05-29 13:24:55 -04005551 ext4_update_inode_fsync_trans(handle, inode, 1);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005552
5553out_stop:
5554 ext4_journal_stop(handle);
Jan Karaea3d7202015-12-07 14:28:03 -05005555out_mmap:
5556 up_write(&EXT4_I(inode)->i_mmap_sem);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005557out_mutex:
Al Viro59551022016-01-22 15:40:57 -05005558 inode_unlock(inode);
Namjae Jeon9eb79482014-02-23 15:18:59 -05005559 return ret;
5560}
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005561
Namjae Jeon331573f2015-06-09 01:55:03 -04005562/*
5563 * ext4_insert_range:
5564 * This function implements the FALLOC_FL_INSERT_RANGE flag of fallocate.
5565 * The data blocks starting from @offset to the EOF are shifted by @len
5566 * towards right to create a hole in the @inode. Inode size is increased
5567 * by len bytes.
5568 * Returns 0 on success, error otherwise.
5569 */
Eric Biggers43f81672019-12-31 12:04:40 -06005570static int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len)
Namjae Jeon331573f2015-06-09 01:55:03 -04005571{
5572 struct super_block *sb = inode->i_sb;
5573 handle_t *handle;
5574 struct ext4_ext_path *path;
5575 struct ext4_extent *extent;
5576 ext4_lblk_t offset_lblk, len_lblk, ee_start_lblk = 0;
5577 unsigned int credits, ee_len;
5578 int ret = 0, depth, split_flag = 0;
5579 loff_t ioffset;
5580
5581 /*
5582 * We need to test this early because xfstests assumes that an
5583 * insert range of (0, 1) will return EOPNOTSUPP if the file
5584 * system does not support insert range.
5585 */
5586 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5587 return -EOPNOTSUPP;
5588
Eric Biggers9b02e492019-12-31 12:04:38 -06005589 /* Insert range works only on fs cluster size aligned regions. */
5590 if (!IS_ALIGNED(offset | len, EXT4_CLUSTER_SIZE(sb)))
Namjae Jeon331573f2015-06-09 01:55:03 -04005591 return -EINVAL;
5592
Namjae Jeon331573f2015-06-09 01:55:03 -04005593 trace_ext4_insert_range(inode, offset, len);
5594
5595 offset_lblk = offset >> EXT4_BLOCK_SIZE_BITS(sb);
5596 len_lblk = len >> EXT4_BLOCK_SIZE_BITS(sb);
5597
5598 /* Call ext4_force_commit to flush all data in case of data=journal */
5599 if (ext4_should_journal_data(inode)) {
5600 ret = ext4_force_commit(inode->i_sb);
5601 if (ret)
5602 return ret;
5603 }
5604
Al Viro59551022016-01-22 15:40:57 -05005605 inode_lock(inode);
Namjae Jeon331573f2015-06-09 01:55:03 -04005606 /* Currently just for extent based files */
5607 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
5608 ret = -EOPNOTSUPP;
5609 goto out_mutex;
5610 }
5611
Eric Biggers9b02e492019-12-31 12:04:38 -06005612 /* Check whether the maximum file size would be exceeded */
5613 if (len > inode->i_sb->s_maxbytes - inode->i_size) {
Namjae Jeon331573f2015-06-09 01:55:03 -04005614 ret = -EFBIG;
5615 goto out_mutex;
5616 }
5617
Eric Biggers9b02e492019-12-31 12:04:38 -06005618 /* Offset must be less than i_size */
5619 if (offset >= inode->i_size) {
Namjae Jeon331573f2015-06-09 01:55:03 -04005620 ret = -EINVAL;
5621 goto out_mutex;
5622 }
5623
Namjae Jeon331573f2015-06-09 01:55:03 -04005624 /* Wait for existing dio to complete */
Namjae Jeon331573f2015-06-09 01:55:03 -04005625 inode_dio_wait(inode);
5626
Jan Karaea3d7202015-12-07 14:28:03 -05005627 /*
5628 * Prevent page faults from reinstantiating pages we have released from
5629 * page cache.
5630 */
5631 down_write(&EXT4_I(inode)->i_mmap_sem);
Ross Zwisler430657b2018-07-29 17:00:22 -04005632
5633 ret = ext4_break_layouts(inode);
5634 if (ret)
5635 goto out_mmap;
5636
Jan Kara32ebffd2015-12-07 14:31:11 -05005637 /*
5638 * Need to round down to align start offset to page size boundary
5639 * for page size > block size.
5640 */
5641 ioffset = round_down(offset, PAGE_SIZE);
5642 /* Write out all dirty pages */
5643 ret = filemap_write_and_wait_range(inode->i_mapping, ioffset,
5644 LLONG_MAX);
5645 if (ret)
5646 goto out_mmap;
Jan Karaea3d7202015-12-07 14:28:03 -05005647 truncate_pagecache(inode, ioffset);
5648
Namjae Jeon331573f2015-06-09 01:55:03 -04005649 credits = ext4_writepage_trans_blocks(inode);
5650 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
5651 if (IS_ERR(handle)) {
5652 ret = PTR_ERR(handle);
Jan Karaea3d7202015-12-07 14:28:03 -05005653 goto out_mmap;
Namjae Jeon331573f2015-06-09 01:55:03 -04005654 }
5655
5656 /* Expand file to avoid data loss if there is error while shifting */
5657 inode->i_size += len;
5658 EXT4_I(inode)->i_disksize += len;
Deepa Dinamanieeca7ea2016-11-14 21:40:10 -05005659 inode->i_mtime = inode->i_ctime = current_time(inode);
Namjae Jeon331573f2015-06-09 01:55:03 -04005660 ret = ext4_mark_inode_dirty(handle, inode);
5661 if (ret)
5662 goto out_stop;
5663
5664 down_write(&EXT4_I(inode)->i_data_sem);
5665 ext4_discard_preallocations(inode);
5666
5667 path = ext4_find_extent(inode, offset_lblk, NULL, 0);
5668 if (IS_ERR(path)) {
5669 up_write(&EXT4_I(inode)->i_data_sem);
5670 goto out_stop;
5671 }
5672
5673 depth = ext_depth(inode);
5674 extent = path[depth].p_ext;
5675 if (extent) {
5676 ee_start_lblk = le32_to_cpu(extent->ee_block);
5677 ee_len = ext4_ext_get_actual_len(extent);
5678
5679 /*
5680 * If offset_lblk is not the starting block of extent, split
5681 * the extent @offset_lblk
5682 */
5683 if ((offset_lblk > ee_start_lblk) &&
5684 (offset_lblk < (ee_start_lblk + ee_len))) {
5685 if (ext4_ext_is_unwritten(extent))
5686 split_flag = EXT4_EXT_MARK_UNWRIT1 |
5687 EXT4_EXT_MARK_UNWRIT2;
5688 ret = ext4_split_extent_at(handle, inode, &path,
5689 offset_lblk, split_flag,
5690 EXT4_EX_NOCACHE |
5691 EXT4_GET_BLOCKS_PRE_IO |
5692 EXT4_GET_BLOCKS_METADATA_NOFAIL);
5693 }
5694
5695 ext4_ext_drop_refs(path);
5696 kfree(path);
5697 if (ret < 0) {
5698 up_write(&EXT4_I(inode)->i_data_sem);
5699 goto out_stop;
5700 }
Fabian Frederickedf15aa12016-09-15 11:39:52 -04005701 } else {
5702 ext4_ext_drop_refs(path);
5703 kfree(path);
Namjae Jeon331573f2015-06-09 01:55:03 -04005704 }
5705
5706 ret = ext4_es_remove_extent(inode, offset_lblk,
5707 EXT_MAX_BLOCKS - offset_lblk);
5708 if (ret) {
5709 up_write(&EXT4_I(inode)->i_data_sem);
5710 goto out_stop;
5711 }
5712
5713 /*
5714 * if offset_lblk lies in a hole which is at start of file, use
5715 * ee_start_lblk to shift extents
5716 */
5717 ret = ext4_ext_shift_extents(inode, handle,
5718 ee_start_lblk > offset_lblk ? ee_start_lblk : offset_lblk,
5719 len_lblk, SHIFT_RIGHT);
5720
5721 up_write(&EXT4_I(inode)->i_data_sem);
5722 if (IS_SYNC(inode))
5723 ext4_handle_sync(handle);
Jan Kara67a7d5f2017-05-29 13:24:55 -04005724 if (ret >= 0)
5725 ext4_update_inode_fsync_trans(handle, inode, 1);
Namjae Jeon331573f2015-06-09 01:55:03 -04005726
5727out_stop:
5728 ext4_journal_stop(handle);
Jan Karaea3d7202015-12-07 14:28:03 -05005729out_mmap:
5730 up_write(&EXT4_I(inode)->i_mmap_sem);
Namjae Jeon331573f2015-06-09 01:55:03 -04005731out_mutex:
Al Viro59551022016-01-22 15:40:57 -05005732 inode_unlock(inode);
Namjae Jeon331573f2015-06-09 01:55:03 -04005733 return ret;
5734}
5735
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005736/**
Theodore Ts'oc60990b2019-06-19 16:30:03 -04005737 * ext4_swap_extents() - Swap extents between two inodes
5738 * @handle: handle for this transaction
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005739 * @inode1: First inode
5740 * @inode2: Second inode
5741 * @lblk1: Start block for first inode
5742 * @lblk2: Start block for second inode
5743 * @count: Number of blocks to swap
zhenwei.pidcae0582018-03-26 01:44:03 -04005744 * @unwritten: Mark second inode's extents as unwritten after swap
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005745 * @erp: Pointer to save error value
5746 *
5747 * This helper routine does exactly what is promise "swap extents". All other
5748 * stuff such as page-cache locking consistency, bh mapping consistency or
5749 * extent's data copying must be performed by caller.
5750 * Locking:
5751 * i_mutex is held for both inodes
5752 * i_data_sem is locked for write for both inodes
5753 * Assumptions:
5754 * All pages from requested range are locked for both inodes
5755 */
5756int
5757ext4_swap_extents(handle_t *handle, struct inode *inode1,
zhenwei.pidcae0582018-03-26 01:44:03 -04005758 struct inode *inode2, ext4_lblk_t lblk1, ext4_lblk_t lblk2,
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005759 ext4_lblk_t count, int unwritten, int *erp)
5760{
5761 struct ext4_ext_path *path1 = NULL;
5762 struct ext4_ext_path *path2 = NULL;
5763 int replaced_count = 0;
5764
5765 BUG_ON(!rwsem_is_locked(&EXT4_I(inode1)->i_data_sem));
5766 BUG_ON(!rwsem_is_locked(&EXT4_I(inode2)->i_data_sem));
Al Viro59551022016-01-22 15:40:57 -05005767 BUG_ON(!inode_is_locked(inode1));
5768 BUG_ON(!inode_is_locked(inode2));
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005769
5770 *erp = ext4_es_remove_extent(inode1, lblk1, count);
Theodore Ts'o19008f62014-08-31 15:03:14 -04005771 if (unlikely(*erp))
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005772 return 0;
5773 *erp = ext4_es_remove_extent(inode2, lblk2, count);
Theodore Ts'o19008f62014-08-31 15:03:14 -04005774 if (unlikely(*erp))
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005775 return 0;
5776
5777 while (count) {
5778 struct ext4_extent *ex1, *ex2, tmp_ex;
5779 ext4_lblk_t e1_blk, e2_blk;
5780 int e1_len, e2_len, len;
5781 int split = 0;
5782
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04005783 path1 = ext4_find_extent(inode1, lblk1, NULL, EXT4_EX_NOCACHE);
Viresh Kumara1c83682015-08-12 15:59:44 +05305784 if (IS_ERR(path1)) {
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005785 *erp = PTR_ERR(path1);
Theodore Ts'o19008f62014-08-31 15:03:14 -04005786 path1 = NULL;
5787 finish:
5788 count = 0;
5789 goto repeat;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005790 }
Theodore Ts'oed8a1a72014-09-01 14:43:09 -04005791 path2 = ext4_find_extent(inode2, lblk2, NULL, EXT4_EX_NOCACHE);
Viresh Kumara1c83682015-08-12 15:59:44 +05305792 if (IS_ERR(path2)) {
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005793 *erp = PTR_ERR(path2);
Theodore Ts'o19008f62014-08-31 15:03:14 -04005794 path2 = NULL;
5795 goto finish;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005796 }
5797 ex1 = path1[path1->p_depth].p_ext;
5798 ex2 = path2[path2->p_depth].p_ext;
5799 /* Do we have somthing to swap ? */
5800 if (unlikely(!ex2 || !ex1))
Theodore Ts'o19008f62014-08-31 15:03:14 -04005801 goto finish;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005802
5803 e1_blk = le32_to_cpu(ex1->ee_block);
5804 e2_blk = le32_to_cpu(ex2->ee_block);
5805 e1_len = ext4_ext_get_actual_len(ex1);
5806 e2_len = ext4_ext_get_actual_len(ex2);
5807
5808 /* Hole handling */
5809 if (!in_range(lblk1, e1_blk, e1_len) ||
5810 !in_range(lblk2, e2_blk, e2_len)) {
5811 ext4_lblk_t next1, next2;
5812
5813 /* if hole after extent, then go to next extent */
5814 next1 = ext4_ext_next_allocated_block(path1);
5815 next2 = ext4_ext_next_allocated_block(path2);
5816 /* If hole before extent, then shift to that extent */
5817 if (e1_blk > lblk1)
5818 next1 = e1_blk;
5819 if (e2_blk > lblk2)
Maninder Singh4e562012017-08-06 01:33:07 -04005820 next2 = e2_blk;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005821 /* Do we have something to swap */
5822 if (next1 == EXT_MAX_BLOCKS || next2 == EXT_MAX_BLOCKS)
Theodore Ts'o19008f62014-08-31 15:03:14 -04005823 goto finish;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005824 /* Move to the rightest boundary */
5825 len = next1 - lblk1;
5826 if (len < next2 - lblk2)
5827 len = next2 - lblk2;
5828 if (len > count)
5829 len = count;
5830 lblk1 += len;
5831 lblk2 += len;
5832 count -= len;
5833 goto repeat;
5834 }
5835
5836 /* Prepare left boundary */
5837 if (e1_blk < lblk1) {
5838 split = 1;
5839 *erp = ext4_force_split_extent_at(handle, inode1,
Theodore Ts'odfe50802014-09-01 14:37:09 -04005840 &path1, lblk1, 0);
Theodore Ts'o19008f62014-08-31 15:03:14 -04005841 if (unlikely(*erp))
5842 goto finish;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005843 }
5844 if (e2_blk < lblk2) {
5845 split = 1;
5846 *erp = ext4_force_split_extent_at(handle, inode2,
Theodore Ts'odfe50802014-09-01 14:37:09 -04005847 &path2, lblk2, 0);
Theodore Ts'o19008f62014-08-31 15:03:14 -04005848 if (unlikely(*erp))
5849 goto finish;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005850 }
Theodore Ts'odfe50802014-09-01 14:37:09 -04005851 /* ext4_split_extent_at() may result in leaf extent split,
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005852 * path must to be revalidated. */
5853 if (split)
5854 goto repeat;
5855
5856 /* Prepare right boundary */
5857 len = count;
5858 if (len > e1_blk + e1_len - lblk1)
5859 len = e1_blk + e1_len - lblk1;
5860 if (len > e2_blk + e2_len - lblk2)
5861 len = e2_blk + e2_len - lblk2;
5862
5863 if (len != e1_len) {
5864 split = 1;
5865 *erp = ext4_force_split_extent_at(handle, inode1,
Theodore Ts'odfe50802014-09-01 14:37:09 -04005866 &path1, lblk1 + len, 0);
Theodore Ts'o19008f62014-08-31 15:03:14 -04005867 if (unlikely(*erp))
5868 goto finish;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005869 }
5870 if (len != e2_len) {
5871 split = 1;
5872 *erp = ext4_force_split_extent_at(handle, inode2,
Theodore Ts'odfe50802014-09-01 14:37:09 -04005873 &path2, lblk2 + len, 0);
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005874 if (*erp)
Theodore Ts'o19008f62014-08-31 15:03:14 -04005875 goto finish;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005876 }
Theodore Ts'odfe50802014-09-01 14:37:09 -04005877 /* ext4_split_extent_at() may result in leaf extent split,
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005878 * path must to be revalidated. */
5879 if (split)
5880 goto repeat;
5881
5882 BUG_ON(e2_len != e1_len);
5883 *erp = ext4_ext_get_access(handle, inode1, path1 + path1->p_depth);
Theodore Ts'o19008f62014-08-31 15:03:14 -04005884 if (unlikely(*erp))
5885 goto finish;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005886 *erp = ext4_ext_get_access(handle, inode2, path2 + path2->p_depth);
Theodore Ts'o19008f62014-08-31 15:03:14 -04005887 if (unlikely(*erp))
5888 goto finish;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005889
5890 /* Both extents are fully inside boundaries. Swap it now */
5891 tmp_ex = *ex1;
5892 ext4_ext_store_pblock(ex1, ext4_ext_pblock(ex2));
5893 ext4_ext_store_pblock(ex2, ext4_ext_pblock(&tmp_ex));
5894 ex1->ee_len = cpu_to_le16(e2_len);
5895 ex2->ee_len = cpu_to_le16(e1_len);
5896 if (unwritten)
5897 ext4_ext_mark_unwritten(ex2);
5898 if (ext4_ext_is_unwritten(&tmp_ex))
5899 ext4_ext_mark_unwritten(ex1);
5900
5901 ext4_ext_try_to_merge(handle, inode2, path2, ex2);
5902 ext4_ext_try_to_merge(handle, inode1, path1, ex1);
5903 *erp = ext4_ext_dirty(handle, inode2, path2 +
5904 path2->p_depth);
Theodore Ts'o19008f62014-08-31 15:03:14 -04005905 if (unlikely(*erp))
5906 goto finish;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005907 *erp = ext4_ext_dirty(handle, inode1, path1 +
5908 path1->p_depth);
5909 /*
5910 * Looks scarry ah..? second inode already points to new blocks,
5911 * and it was successfully dirtied. But luckily error may happen
5912 * only due to journal error, so full transaction will be
5913 * aborted anyway.
5914 */
Theodore Ts'o19008f62014-08-31 15:03:14 -04005915 if (unlikely(*erp))
5916 goto finish;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005917 lblk1 += len;
5918 lblk2 += len;
5919 replaced_count += len;
5920 count -= len;
5921
5922 repeat:
Theodore Ts'ob7ea89a2014-09-01 14:39:09 -04005923 ext4_ext_drop_refs(path1);
5924 kfree(path1);
5925 ext4_ext_drop_refs(path2);
5926 kfree(path2);
5927 path1 = path2 = NULL;
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005928 }
Dmitry Monakhovfcf6b1b72014-08-30 23:52:19 -04005929 return replaced_count;
5930}
Eric Whitney0b02f4c2018-10-01 14:19:37 -04005931
5932/*
5933 * ext4_clu_mapped - determine whether any block in a logical cluster has
5934 * been mapped to a physical cluster
5935 *
5936 * @inode - file containing the logical cluster
5937 * @lclu - logical cluster of interest
5938 *
5939 * Returns 1 if any block in the logical cluster is mapped, signifying
5940 * that a physical cluster has been allocated for it. Otherwise,
5941 * returns 0. Can also return negative error codes. Derived from
5942 * ext4_ext_map_blocks().
5943 */
5944int ext4_clu_mapped(struct inode *inode, ext4_lblk_t lclu)
5945{
5946 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5947 struct ext4_ext_path *path;
5948 int depth, mapped = 0, err = 0;
5949 struct ext4_extent *extent;
5950 ext4_lblk_t first_lblk, first_lclu, last_lclu;
5951
5952 /* search for the extent closest to the first block in the cluster */
5953 path = ext4_find_extent(inode, EXT4_C2B(sbi, lclu), NULL, 0);
5954 if (IS_ERR(path)) {
5955 err = PTR_ERR(path);
5956 path = NULL;
5957 goto out;
5958 }
5959
5960 depth = ext_depth(inode);
5961
5962 /*
5963 * A consistent leaf must not be empty. This situation is possible,
5964 * though, _during_ tree modification, and it's why an assert can't
5965 * be put in ext4_find_extent().
5966 */
5967 if (unlikely(path[depth].p_ext == NULL && depth != 0)) {
5968 EXT4_ERROR_INODE(inode,
5969 "bad extent address - lblock: %lu, depth: %d, pblock: %lld",
5970 (unsigned long) EXT4_C2B(sbi, lclu),
5971 depth, path[depth].p_block);
5972 err = -EFSCORRUPTED;
5973 goto out;
5974 }
5975
5976 extent = path[depth].p_ext;
5977
5978 /* can't be mapped if the extent tree is empty */
5979 if (extent == NULL)
5980 goto out;
5981
5982 first_lblk = le32_to_cpu(extent->ee_block);
5983 first_lclu = EXT4_B2C(sbi, first_lblk);
5984
5985 /*
5986 * Three possible outcomes at this point - found extent spanning
5987 * the target cluster, to the left of the target cluster, or to the
5988 * right of the target cluster. The first two cases are handled here.
5989 * The last case indicates the target cluster is not mapped.
5990 */
5991 if (lclu >= first_lclu) {
5992 last_lclu = EXT4_B2C(sbi, first_lblk +
5993 ext4_ext_get_actual_len(extent) - 1);
5994 if (lclu <= last_lclu) {
5995 mapped = 1;
5996 } else {
5997 first_lblk = ext4_ext_next_allocated_block(path);
5998 first_lclu = EXT4_B2C(sbi, first_lblk);
5999 if (lclu == first_lclu)
6000 mapped = 1;
6001 }
6002 }
6003
6004out:
6005 ext4_ext_drop_refs(path);
6006 kfree(path);
6007
6008 return err ? err : mapped;
6009}