a81eca6434dd77b9647f52d8d2e2e301560ee248
[linux-2.6.git] / drivers / md / raid5.c
1 /*
2  * raid5.c : Multiple Devices driver for Linux
3  *         Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
4  *         Copyright (C) 1999, 2000 Ingo Molnar
5  *         Copyright (C) 2002, 2003 H. Peter Anvin
6  *
7  * RAID-4/5/6 management functions.
8  * Thanks to Penguin Computing for making the RAID-6 development possible
9  * by donating a test server!
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2, or (at your option)
14  * any later version.
15  *
16  * You should have received a copy of the GNU General Public License
17  * (for example /usr/src/linux/COPYING); if not, write to the Free
18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19  */
20
21 /*
22  * BITMAP UNPLUGGING:
23  *
24  * The sequencing for updating the bitmap reliably is a little
25  * subtle (and I got it wrong the first time) so it deserves some
26  * explanation.
27  *
28  * We group bitmap updates into batches.  Each batch has a number.
29  * We may write out several batches at once, but that isn't very important.
30  * conf->seq_write is the number of the last batch successfully written.
31  * conf->seq_flush is the number of the last batch that was closed to
32  *    new additions.
33  * When we discover that we will need to write to any block in a stripe
34  * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
35  * the number of the batch it will be in. This is seq_flush+1.
36  * When we are ready to do a write, if that batch hasn't been written yet,
37  *   we plug the array and queue the stripe for later.
38  * When an unplug happens, we increment bm_flush, thus closing the current
39  *   batch.
40  * When we notice that bm_flush > bm_write, we write out all pending updates
41  * to the bitmap, and advance bm_write to where bm_flush was.
42  * This may occasionally write a bit out twice, but is sure never to
43  * miss any bits.
44  */
45
46 #include <linux/blkdev.h>
47 #include <linux/kthread.h>
48 #include <linux/raid/pq.h>
49 #include <linux/async_tx.h>
50 #include <linux/async.h>
51 #include <linux/seq_file.h>
52 #include <linux/cpu.h>
53 #include <linux/slab.h>
54 #include <linux/ratelimit.h>
55 #include "md.h"
56 #include "raid5.h"
57 #include "raid0.h"
58 #include "bitmap.h"
59
60 /*
61  * Stripe cache
62  */
63
64 #define NR_STRIPES              256
65 #define STRIPE_SIZE             PAGE_SIZE
66 #define STRIPE_SHIFT            (PAGE_SHIFT - 9)
67 #define STRIPE_SECTORS          (STRIPE_SIZE>>9)
68 #define IO_THRESHOLD            1
69 #define BYPASS_THRESHOLD        1
70 #define NR_HASH                 (PAGE_SIZE / sizeof(struct hlist_head))
71 #define HASH_MASK               (NR_HASH - 1)
72
73 #define stripe_hash(conf, sect) (&((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK]))
74
75 /* bio's attached to a stripe+device for I/O are linked together in bi_sector
76  * order without overlap.  There may be several bio's per stripe+device, and
77  * a bio could span several devices.
78  * When walking this list for a particular stripe+device, we must never proceed
79  * beyond a bio that extends past this device, as the next bio might no longer
80  * be valid.
81  * This macro is used to determine the 'next' bio in the list, given the sector
82  * of the current stripe+device
83  */
84 #define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL)
85 /*
86  * The following can be used to debug the driver
87  */
88 #define RAID5_PARANOIA  1
89 #if RAID5_PARANOIA && defined(CONFIG_SMP)
90 # define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock)
91 #else
92 # define CHECK_DEVLOCK()
93 #endif
94
95 #ifdef DEBUG
96 #define inline
97 #define __inline__
98 #endif
99
100 /*
101  * We maintain a biased count of active stripes in the bottom 16 bits of
102  * bi_phys_segments, and a count of processed stripes in the upper 16 bits
103  */
104 static inline int raid5_bi_phys_segments(struct bio *bio)
105 {
106         return bio->bi_phys_segments & 0xffff;
107 }
108
109 static inline int raid5_bi_hw_segments(struct bio *bio)
110 {
111         return (bio->bi_phys_segments >> 16) & 0xffff;
112 }
113
114 static inline int raid5_dec_bi_phys_segments(struct bio *bio)
115 {
116         --bio->bi_phys_segments;
117         return raid5_bi_phys_segments(bio);
118 }
119
120 static inline int raid5_dec_bi_hw_segments(struct bio *bio)
121 {
122         unsigned short val = raid5_bi_hw_segments(bio);
123
124         --val;
125         bio->bi_phys_segments = (val << 16) | raid5_bi_phys_segments(bio);
126         return val;
127 }
128
129 static inline void raid5_set_bi_hw_segments(struct bio *bio, unsigned int cnt)
130 {
131         bio->bi_phys_segments = raid5_bi_phys_segments(bio) | (cnt << 16);
132 }
133
134 /* Find first data disk in a raid6 stripe */
135 static inline int raid6_d0(struct stripe_head *sh)
136 {
137         if (sh->ddf_layout)
138                 /* ddf always start from first device */
139                 return 0;
140         /* md starts just after Q block */
141         if (sh->qd_idx == sh->disks - 1)
142                 return 0;
143         else
144                 return sh->qd_idx + 1;
145 }
146 static inline int raid6_next_disk(int disk, int raid_disks)
147 {
148         disk++;
149         return (disk < raid_disks) ? disk : 0;
150 }
151
152 /* When walking through the disks in a raid5, starting at raid6_d0,
153  * We need to map each disk to a 'slot', where the data disks are slot
154  * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
155  * is raid_disks-1.  This help does that mapping.
156  */
157 static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
158                              int *count, int syndrome_disks)
159 {
160         int slot = *count;
161
162         if (sh->ddf_layout)
163                 (*count)++;
164         if (idx == sh->pd_idx)
165                 return syndrome_disks;
166         if (idx == sh->qd_idx)
167                 return syndrome_disks + 1;
168         if (!sh->ddf_layout)
169                 (*count)++;
170         return slot;
171 }
172
173 static void return_io(struct bio *return_bi)
174 {
175         struct bio *bi = return_bi;
176         while (bi) {
177
178                 return_bi = bi->bi_next;
179                 bi->bi_next = NULL;
180                 bi->bi_size = 0;
181                 bio_endio(bi, 0);
182                 bi = return_bi;
183         }
184 }
185
186 static void print_raid5_conf (raid5_conf_t *conf);
187
188 static int stripe_operations_active(struct stripe_head *sh)
189 {
190         return sh->check_state || sh->reconstruct_state ||
191                test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
192                test_bit(STRIPE_COMPUTE_RUN, &sh->state);
193 }
194
195 static void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh)
196 {
197         if (atomic_dec_and_test(&sh->count)) {
198                 BUG_ON(!list_empty(&sh->lru));
199                 BUG_ON(atomic_read(&conf->active_stripes)==0);
200                 if (test_bit(STRIPE_HANDLE, &sh->state)) {
201                         if (test_bit(STRIPE_DELAYED, &sh->state))
202                                 list_add_tail(&sh->lru, &conf->delayed_list);
203                         else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
204                                    sh->bm_seq - conf->seq_write > 0)
205                                 list_add_tail(&sh->lru, &conf->bitmap_list);
206                         else {
207                                 clear_bit(STRIPE_BIT_DELAY, &sh->state);
208                                 list_add_tail(&sh->lru, &conf->handle_list);
209                         }
210                         md_wakeup_thread(conf->mddev->thread);
211                 } else {
212                         BUG_ON(stripe_operations_active(sh));
213                         if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
214                                 atomic_dec(&conf->preread_active_stripes);
215                                 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD)
216                                         md_wakeup_thread(conf->mddev->thread);
217                         }
218                         atomic_dec(&conf->active_stripes);
219                         if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
220                                 list_add_tail(&sh->lru, &conf->inactive_list);
221                                 wake_up(&conf->wait_for_stripe);
222                                 if (conf->retry_read_aligned)
223                                         md_wakeup_thread(conf->mddev->thread);
224                         }
225                 }
226         }
227 }
228
229 static void release_stripe(struct stripe_head *sh)
230 {
231         raid5_conf_t *conf = sh->raid_conf;
232         unsigned long flags;
233
234         spin_lock_irqsave(&conf->device_lock, flags);
235         __release_stripe(conf, sh);
236         spin_unlock_irqrestore(&conf->device_lock, flags);
237 }
238
239 static inline void remove_hash(struct stripe_head *sh)
240 {
241         pr_debug("remove_hash(), stripe %llu\n",
242                 (unsigned long long)sh->sector);
243
244         hlist_del_init(&sh->hash);
245 }
246
247 static inline void insert_hash(raid5_conf_t *conf, struct stripe_head *sh)
248 {
249         struct hlist_head *hp = stripe_hash(conf, sh->sector);
250
251         pr_debug("insert_hash(), stripe %llu\n",
252                 (unsigned long long)sh->sector);
253
254         CHECK_DEVLOCK();
255         hlist_add_head(&sh->hash, hp);
256 }
257
258
259 /* find an idle stripe, make sure it is unhashed, and return it. */
260 static struct stripe_head *get_free_stripe(raid5_conf_t *conf)
261 {
262         struct stripe_head *sh = NULL;
263         struct list_head *first;
264
265         CHECK_DEVLOCK();
266         if (list_empty(&conf->inactive_list))
267                 goto out;
268         first = conf->inactive_list.next;
269         sh = list_entry(first, struct stripe_head, lru);
270         list_del_init(first);
271         remove_hash(sh);
272         atomic_inc(&conf->active_stripes);
273 out:
274         return sh;
275 }
276
277 static void shrink_buffers(struct stripe_head *sh)
278 {
279         struct page *p;
280         int i;
281         int num = sh->raid_conf->pool_size;
282
283         for (i = 0; i < num ; i++) {
284                 p = sh->dev[i].page;
285                 if (!p)
286                         continue;
287                 sh->dev[i].page = NULL;
288                 put_page(p);
289         }
290 }
291
292 static int grow_buffers(struct stripe_head *sh)
293 {
294         int i;
295         int num = sh->raid_conf->pool_size;
296
297         for (i = 0; i < num; i++) {
298                 struct page *page;
299
300                 if (!(page = alloc_page(GFP_KERNEL))) {
301                         return 1;
302                 }
303                 sh->dev[i].page = page;
304         }
305         return 0;
306 }
307
308 static void raid5_build_block(struct stripe_head *sh, int i, int previous);
309 static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous,
310                             struct stripe_head *sh);
311
312 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
313 {
314         raid5_conf_t *conf = sh->raid_conf;
315         int i;
316
317         BUG_ON(atomic_read(&sh->count) != 0);
318         BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
319         BUG_ON(stripe_operations_active(sh));
320
321         CHECK_DEVLOCK();
322         pr_debug("init_stripe called, stripe %llu\n",
323                 (unsigned long long)sh->sector);
324
325         remove_hash(sh);
326
327         sh->generation = conf->generation - previous;
328         sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
329         sh->sector = sector;
330         stripe_set_idx(sector, conf, previous, sh);
331         sh->state = 0;
332
333
334         for (i = sh->disks; i--; ) {
335                 struct r5dev *dev = &sh->dev[i];
336
337                 if (dev->toread || dev->read || dev->towrite || dev->written ||
338                     test_bit(R5_LOCKED, &dev->flags)) {
339                         printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
340                                (unsigned long long)sh->sector, i, dev->toread,
341                                dev->read, dev->towrite, dev->written,
342                                test_bit(R5_LOCKED, &dev->flags));
343                         BUG();
344                 }
345                 dev->flags = 0;
346                 raid5_build_block(sh, i, previous);
347         }
348         insert_hash(conf, sh);
349 }
350
351 static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector,
352                                          short generation)
353 {
354         struct stripe_head *sh;
355         struct hlist_node *hn;
356
357         CHECK_DEVLOCK();
358         pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
359         hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash)
360                 if (sh->sector == sector && sh->generation == generation)
361                         return sh;
362         pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
363         return NULL;
364 }
365
366 /*
367  * Need to check if array has failed when deciding whether to:
368  *  - start an array
369  *  - remove non-faulty devices
370  *  - add a spare
371  *  - allow a reshape
372  * This determination is simple when no reshape is happening.
373  * However if there is a reshape, we need to carefully check
374  * both the before and after sections.
375  * This is because some failed devices may only affect one
376  * of the two sections, and some non-in_sync devices may
377  * be insync in the section most affected by failed devices.
378  */
379 static int has_failed(raid5_conf_t *conf)
380 {
381         int degraded;
382         int i;
383         if (conf->mddev->reshape_position == MaxSector)
384                 return conf->mddev->degraded > conf->max_degraded;
385
386         rcu_read_lock();
387         degraded = 0;
388         for (i = 0; i < conf->previous_raid_disks; i++) {
389                 mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev);
390                 if (!rdev || test_bit(Faulty, &rdev->flags))
391                         degraded++;
392                 else if (test_bit(In_sync, &rdev->flags))
393                         ;
394                 else
395                         /* not in-sync or faulty.
396                          * If the reshape increases the number of devices,
397                          * this is being recovered by the reshape, so
398                          * this 'previous' section is not in_sync.
399                          * If the number of devices is being reduced however,
400                          * the device can only be part of the array if
401                          * we are reverting a reshape, so this section will
402                          * be in-sync.
403                          */
404                         if (conf->raid_disks >= conf->previous_raid_disks)
405                                 degraded++;
406         }
407         rcu_read_unlock();
408         if (degraded > conf->max_degraded)
409                 return 1;
410         rcu_read_lock();
411         degraded = 0;
412         for (i = 0; i < conf->raid_disks; i++) {
413                 mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev);
414                 if (!rdev || test_bit(Faulty, &rdev->flags))
415                         degraded++;
416                 else if (test_bit(In_sync, &rdev->flags))
417                         ;
418                 else
419                         /* not in-sync or faulty.
420                          * If reshape increases the number of devices, this
421                          * section has already been recovered, else it
422                          * almost certainly hasn't.
423                          */
424                         if (conf->raid_disks <= conf->previous_raid_disks)
425                                 degraded++;
426         }
427         rcu_read_unlock();
428         if (degraded > conf->max_degraded)
429                 return 1;
430         return 0;
431 }
432
433 static struct stripe_head *
434 get_active_stripe(raid5_conf_t *conf, sector_t sector,
435                   int previous, int noblock, int noquiesce)
436 {
437         struct stripe_head *sh;
438
439         pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
440
441         spin_lock_irq(&conf->device_lock);
442
443         do {
444                 wait_event_lock_irq(conf->wait_for_stripe,
445                                     conf->quiesce == 0 || noquiesce,
446                                     conf->device_lock, /* nothing */);
447                 sh = __find_stripe(conf, sector, conf->generation - previous);
448                 if (!sh) {
449                         if (!conf->inactive_blocked)
450                                 sh = get_free_stripe(conf);
451                         if (noblock && sh == NULL)
452                                 break;
453                         if (!sh) {
454                                 conf->inactive_blocked = 1;
455                                 wait_event_lock_irq(conf->wait_for_stripe,
456                                                     !list_empty(&conf->inactive_list) &&
457                                                     (atomic_read(&conf->active_stripes)
458                                                      < (conf->max_nr_stripes *3/4)
459                                                      || !conf->inactive_blocked),
460                                                     conf->device_lock,
461                                                     );
462                                 conf->inactive_blocked = 0;
463                         } else
464                                 init_stripe(sh, sector, previous);
465                 } else {
466                         if (atomic_read(&sh->count)) {
467                                 BUG_ON(!list_empty(&sh->lru)
468                                     && !test_bit(STRIPE_EXPANDING, &sh->state));
469                         } else {
470                                 if (!test_bit(STRIPE_HANDLE, &sh->state))
471                                         atomic_inc(&conf->active_stripes);
472                                 if (list_empty(&sh->lru) &&
473                                     !test_bit(STRIPE_EXPANDING, &sh->state))
474                                         BUG();
475                                 list_del_init(&sh->lru);
476                         }
477                 }
478         } while (sh == NULL);
479
480         if (sh)
481                 atomic_inc(&sh->count);
482
483         spin_unlock_irq(&conf->device_lock);
484         return sh;
485 }
486
487 static void
488 raid5_end_read_request(struct bio *bi, int error);
489 static void
490 raid5_end_write_request(struct bio *bi, int error);
491
492 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
493 {
494         raid5_conf_t *conf = sh->raid_conf;
495         int i, disks = sh->disks;
496
497         might_sleep();
498
499         for (i = disks; i--; ) {
500                 int rw;
501                 struct bio *bi;
502                 mdk_rdev_t *rdev;
503                 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
504                         if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
505                                 rw = WRITE_FUA;
506                         else
507                                 rw = WRITE;
508                 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
509                         rw = READ;
510                 else
511                         continue;
512
513                 bi = &sh->dev[i].req;
514
515                 bi->bi_rw = rw;
516                 if (rw & WRITE)
517                         bi->bi_end_io = raid5_end_write_request;
518                 else
519                         bi->bi_end_io = raid5_end_read_request;
520
521                 rcu_read_lock();
522                 rdev = rcu_dereference(conf->disks[i].rdev);
523                 if (rdev && test_bit(Faulty, &rdev->flags))
524                         rdev = NULL;
525                 if (rdev)
526                         atomic_inc(&rdev->nr_pending);
527                 rcu_read_unlock();
528
529                 if (rdev) {
530                         if (s->syncing || s->expanding || s->expanded)
531                                 md_sync_acct(rdev->bdev, STRIPE_SECTORS);
532
533                         set_bit(STRIPE_IO_STARTED, &sh->state);
534
535                         bi->bi_bdev = rdev->bdev;
536                         pr_debug("%s: for %llu schedule op %ld on disc %d\n",
537                                 __func__, (unsigned long long)sh->sector,
538                                 bi->bi_rw, i);
539                         atomic_inc(&sh->count);
540                         bi->bi_sector = sh->sector + rdev->data_offset;
541                         bi->bi_flags = 1 << BIO_UPTODATE;
542                         bi->bi_vcnt = 1;
543                         bi->bi_max_vecs = 1;
544                         bi->bi_idx = 0;
545                         bi->bi_io_vec = &sh->dev[i].vec;
546                         bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
547                         bi->bi_io_vec[0].bv_offset = 0;
548                         bi->bi_size = STRIPE_SIZE;
549                         bi->bi_next = NULL;
550                         generic_make_request(bi);
551                 } else {
552                         if (rw & WRITE)
553                                 set_bit(STRIPE_DEGRADED, &sh->state);
554                         pr_debug("skip op %ld on disc %d for sector %llu\n",
555                                 bi->bi_rw, i, (unsigned long long)sh->sector);
556                         clear_bit(R5_LOCKED, &sh->dev[i].flags);
557                         set_bit(STRIPE_HANDLE, &sh->state);
558                 }
559         }
560 }
561
562 static struct dma_async_tx_descriptor *
563 async_copy_data(int frombio, struct bio *bio, struct page *page,
564         sector_t sector, struct dma_async_tx_descriptor *tx)
565 {
566         struct bio_vec *bvl;
567         struct page *bio_page;
568         int i;
569         int page_offset;
570         struct async_submit_ctl submit;
571         enum async_tx_flags flags = 0;
572
573         if (bio->bi_sector >= sector)
574                 page_offset = (signed)(bio->bi_sector - sector) * 512;
575         else
576                 page_offset = (signed)(sector - bio->bi_sector) * -512;
577
578         if (frombio)
579                 flags |= ASYNC_TX_FENCE;
580         init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
581
582         bio_for_each_segment(bvl, bio, i) {
583                 int len = bvl->bv_len;
584                 int clen;
585                 int b_offset = 0;
586
587                 if (page_offset < 0) {
588                         b_offset = -page_offset;
589                         page_offset += b_offset;
590                         len -= b_offset;
591                 }
592
593                 if (len > 0 && page_offset + len > STRIPE_SIZE)
594                         clen = STRIPE_SIZE - page_offset;
595                 else
596                         clen = len;
597
598                 if (clen > 0) {
599                         b_offset += bvl->bv_offset;
600                         bio_page = bvl->bv_page;
601                         if (frombio)
602                                 tx = async_memcpy(page, bio_page, page_offset,
603                                                   b_offset, clen, &submit);
604                         else
605                                 tx = async_memcpy(bio_page, page, b_offset,
606                                                   page_offset, clen, &submit);
607                 }
608                 /* chain the operations */
609                 submit.depend_tx = tx;
610
611                 if (clen < len) /* hit end of page */
612                         break;
613                 page_offset +=  len;
614         }
615
616         return tx;
617 }
618
619 static void ops_complete_biofill(void *stripe_head_ref)
620 {
621         struct stripe_head *sh = stripe_head_ref;
622         struct bio *return_bi = NULL;
623         raid5_conf_t *conf = sh->raid_conf;
624         int i;
625
626         pr_debug("%s: stripe %llu\n", __func__,
627                 (unsigned long long)sh->sector);
628
629         /* clear completed biofills */
630         spin_lock_irq(&conf->device_lock);
631         for (i = sh->disks; i--; ) {
632                 struct r5dev *dev = &sh->dev[i];
633
634                 /* acknowledge completion of a biofill operation */
635                 /* and check if we need to reply to a read request,
636                  * new R5_Wantfill requests are held off until
637                  * !STRIPE_BIOFILL_RUN
638                  */
639                 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
640                         struct bio *rbi, *rbi2;
641
642                         BUG_ON(!dev->read);
643                         rbi = dev->read;
644                         dev->read = NULL;
645                         while (rbi && rbi->bi_sector <
646                                 dev->sector + STRIPE_SECTORS) {
647                                 rbi2 = r5_next_bio(rbi, dev->sector);
648                                 if (!raid5_dec_bi_phys_segments(rbi)) {
649                                         rbi->bi_next = return_bi;
650                                         return_bi = rbi;
651                                 }
652                                 rbi = rbi2;
653                         }
654                 }
655         }
656         spin_unlock_irq(&conf->device_lock);
657         clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
658
659         return_io(return_bi);
660
661         set_bit(STRIPE_HANDLE, &sh->state);
662         release_stripe(sh);
663 }
664
665 static void ops_run_biofill(struct stripe_head *sh)
666 {
667         struct dma_async_tx_descriptor *tx = NULL;
668         raid5_conf_t *conf = sh->raid_conf;
669         struct async_submit_ctl submit;
670         int i;
671
672         pr_debug("%s: stripe %llu\n", __func__,
673                 (unsigned long long)sh->sector);
674
675         for (i = sh->disks; i--; ) {
676                 struct r5dev *dev = &sh->dev[i];
677                 if (test_bit(R5_Wantfill, &dev->flags)) {
678                         struct bio *rbi;
679                         spin_lock_irq(&conf->device_lock);
680                         dev->read = rbi = dev->toread;
681                         dev->toread = NULL;
682                         spin_unlock_irq(&conf->device_lock);
683                         while (rbi && rbi->bi_sector <
684                                 dev->sector + STRIPE_SECTORS) {
685                                 tx = async_copy_data(0, rbi, dev->page,
686                                         dev->sector, tx);
687                                 rbi = r5_next_bio(rbi, dev->sector);
688                         }
689                 }
690         }
691
692         atomic_inc(&sh->count);
693         init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
694         async_trigger_callback(&submit);
695 }
696
697 static void mark_target_uptodate(struct stripe_head *sh, int target)
698 {
699         struct r5dev *tgt;
700
701         if (target < 0)
702                 return;
703
704         tgt = &sh->dev[target];
705         set_bit(R5_UPTODATE, &tgt->flags);
706         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
707         clear_bit(R5_Wantcompute, &tgt->flags);
708 }
709
710 static void ops_complete_compute(void *stripe_head_ref)
711 {
712         struct stripe_head *sh = stripe_head_ref;
713
714         pr_debug("%s: stripe %llu\n", __func__,
715                 (unsigned long long)sh->sector);
716
717         /* mark the computed target(s) as uptodate */
718         mark_target_uptodate(sh, sh->ops.target);
719         mark_target_uptodate(sh, sh->ops.target2);
720
721         clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
722         if (sh->check_state == check_state_compute_run)
723                 sh->check_state = check_state_compute_result;
724         set_bit(STRIPE_HANDLE, &sh->state);
725         release_stripe(sh);
726 }
727
728 /* return a pointer to the address conversion region of the scribble buffer */
729 static addr_conv_t *to_addr_conv(struct stripe_head *sh,
730                                  struct raid5_percpu *percpu)
731 {
732         return percpu->scribble + sizeof(struct page *) * (sh->disks + 2);
733 }
734
735 static struct dma_async_tx_descriptor *
736 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
737 {
738         int disks = sh->disks;
739         struct page **xor_srcs = percpu->scribble;
740         int target = sh->ops.target;
741         struct r5dev *tgt = &sh->dev[target];
742         struct page *xor_dest = tgt->page;
743         int count = 0;
744         struct dma_async_tx_descriptor *tx;
745         struct async_submit_ctl submit;
746         int i;
747
748         pr_debug("%s: stripe %llu block: %d\n",
749                 __func__, (unsigned long long)sh->sector, target);
750         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
751
752         for (i = disks; i--; )
753                 if (i != target)
754                         xor_srcs[count++] = sh->dev[i].page;
755
756         atomic_inc(&sh->count);
757
758         init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
759                           ops_complete_compute, sh, to_addr_conv(sh, percpu));
760         if (unlikely(count == 1))
761                 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
762         else
763                 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
764
765         return tx;
766 }
767
768 /* set_syndrome_sources - populate source buffers for gen_syndrome
769  * @srcs - (struct page *) array of size sh->disks
770  * @sh - stripe_head to parse
771  *
772  * Populates srcs in proper layout order for the stripe and returns the
773  * 'count' of sources to be used in a call to async_gen_syndrome.  The P
774  * destination buffer is recorded in srcs[count] and the Q destination
775  * is recorded in srcs[count+1]].
776  */
777 static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
778 {
779         int disks = sh->disks;
780         int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
781         int d0_idx = raid6_d0(sh);
782         int count;
783         int i;
784
785         for (i = 0; i < disks; i++)
786                 srcs[i] = NULL;
787
788         count = 0;
789         i = d0_idx;
790         do {
791                 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
792
793                 srcs[slot] = sh->dev[i].page;
794                 i = raid6_next_disk(i, disks);
795         } while (i != d0_idx);
796
797         return syndrome_disks;
798 }
799
800 static struct dma_async_tx_descriptor *
801 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
802 {
803         int disks = sh->disks;
804         struct page **blocks = percpu->scribble;
805         int target;
806         int qd_idx = sh->qd_idx;
807         struct dma_async_tx_descriptor *tx;
808         struct async_submit_ctl submit;
809         struct r5dev *tgt;
810         struct page *dest;
811         int i;
812         int count;
813
814         if (sh->ops.target < 0)
815                 target = sh->ops.target2;
816         else if (sh->ops.target2 < 0)
817                 target = sh->ops.target;
818         else
819                 /* we should only have one valid target */
820                 BUG();
821         BUG_ON(target < 0);
822         pr_debug("%s: stripe %llu block: %d\n",
823                 __func__, (unsigned long long)sh->sector, target);
824
825         tgt = &sh->dev[target];
826         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
827         dest = tgt->page;
828
829         atomic_inc(&sh->count);
830
831         if (target == qd_idx) {
832                 count = set_syndrome_sources(blocks, sh);
833                 blocks[count] = NULL; /* regenerating p is not necessary */
834                 BUG_ON(blocks[count+1] != dest); /* q should already be set */
835                 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
836                                   ops_complete_compute, sh,
837                                   to_addr_conv(sh, percpu));
838                 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
839         } else {
840                 /* Compute any data- or p-drive using XOR */
841                 count = 0;
842                 for (i = disks; i-- ; ) {
843                         if (i == target || i == qd_idx)
844                                 continue;
845                         blocks[count++] = sh->dev[i].page;
846                 }
847
848                 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
849                                   NULL, ops_complete_compute, sh,
850                                   to_addr_conv(sh, percpu));
851                 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
852         }
853
854         return tx;
855 }
856
857 static struct dma_async_tx_descriptor *
858 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
859 {
860         int i, count, disks = sh->disks;
861         int syndrome_disks = sh->ddf_layout ? disks : disks-2;
862         int d0_idx = raid6_d0(sh);
863         int faila = -1, failb = -1;
864         int target = sh->ops.target;
865         int target2 = sh->ops.target2;
866         struct r5dev *tgt = &sh->dev[target];
867         struct r5dev *tgt2 = &sh->dev[target2];
868         struct dma_async_tx_descriptor *tx;
869         struct page **blocks = percpu->scribble;
870         struct async_submit_ctl submit;
871
872         pr_debug("%s: stripe %llu block1: %d block2: %d\n",
873                  __func__, (unsigned long long)sh->sector, target, target2);
874         BUG_ON(target < 0 || target2 < 0);
875         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
876         BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
877
878         /* we need to open-code set_syndrome_sources to handle the
879          * slot number conversion for 'faila' and 'failb'
880          */
881         for (i = 0; i < disks ; i++)
882                 blocks[i] = NULL;
883         count = 0;
884         i = d0_idx;
885         do {
886                 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
887
888                 blocks[slot] = sh->dev[i].page;
889
890                 if (i == target)
891                         faila = slot;
892                 if (i == target2)
893                         failb = slot;
894                 i = raid6_next_disk(i, disks);
895         } while (i != d0_idx);
896
897         BUG_ON(faila == failb);
898         if (failb < faila)
899                 swap(faila, failb);
900         pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
901                  __func__, (unsigned long long)sh->sector, faila, failb);
902
903         atomic_inc(&sh->count);
904
905         if (failb == syndrome_disks+1) {
906                 /* Q disk is one of the missing disks */
907                 if (faila == syndrome_disks) {
908                         /* Missing P+Q, just recompute */
909                         init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
910                                           ops_complete_compute, sh,
911                                           to_addr_conv(sh, percpu));
912                         return async_gen_syndrome(blocks, 0, syndrome_disks+2,
913                                                   STRIPE_SIZE, &submit);
914                 } else {
915                         struct page *dest;
916                         int data_target;
917                         int qd_idx = sh->qd_idx;
918
919                         /* Missing D+Q: recompute D from P, then recompute Q */
920                         if (target == qd_idx)
921                                 data_target = target2;
922                         else
923                                 data_target = target;
924
925                         count = 0;
926                         for (i = disks; i-- ; ) {
927                                 if (i == data_target || i == qd_idx)
928                                         continue;
929                                 blocks[count++] = sh->dev[i].page;
930                         }
931                         dest = sh->dev[data_target].page;
932                         init_async_submit(&submit,
933                                           ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
934                                           NULL, NULL, NULL,
935                                           to_addr_conv(sh, percpu));
936                         tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
937                                        &submit);
938
939                         count = set_syndrome_sources(blocks, sh);
940                         init_async_submit(&submit, ASYNC_TX_FENCE, tx,
941                                           ops_complete_compute, sh,
942                                           to_addr_conv(sh, percpu));
943                         return async_gen_syndrome(blocks, 0, count+2,
944                                                   STRIPE_SIZE, &submit);
945                 }
946         } else {
947                 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
948                                   ops_complete_compute, sh,
949                                   to_addr_conv(sh, percpu));
950                 if (failb == syndrome_disks) {
951                         /* We're missing D+P. */
952                         return async_raid6_datap_recov(syndrome_disks+2,
953                                                        STRIPE_SIZE, faila,
954                                                        blocks, &submit);
955                 } else {
956                         /* We're missing D+D. */
957                         return async_raid6_2data_recov(syndrome_disks+2,
958                                                        STRIPE_SIZE, faila, failb,
959                                                        blocks, &submit);
960                 }
961         }
962 }
963
964
965 static void ops_complete_prexor(void *stripe_head_ref)
966 {
967         struct stripe_head *sh = stripe_head_ref;
968
969         pr_debug("%s: stripe %llu\n", __func__,
970                 (unsigned long long)sh->sector);
971 }
972
973 static struct dma_async_tx_descriptor *
974 ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
975                struct dma_async_tx_descriptor *tx)
976 {
977         int disks = sh->disks;
978         struct page **xor_srcs = percpu->scribble;
979         int count = 0, pd_idx = sh->pd_idx, i;
980         struct async_submit_ctl submit;
981
982         /* existing parity data subtracted */
983         struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
984
985         pr_debug("%s: stripe %llu\n", __func__,
986                 (unsigned long long)sh->sector);
987
988         for (i = disks; i--; ) {
989                 struct r5dev *dev = &sh->dev[i];
990                 /* Only process blocks that are known to be uptodate */
991                 if (test_bit(R5_Wantdrain, &dev->flags))
992                         xor_srcs[count++] = dev->page;
993         }
994
995         init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
996                           ops_complete_prexor, sh, to_addr_conv(sh, percpu));
997         tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
998
999         return tx;
1000 }
1001
1002 static struct dma_async_tx_descriptor *
1003 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
1004 {
1005         int disks = sh->disks;
1006         int i;
1007
1008         pr_debug("%s: stripe %llu\n", __func__,
1009                 (unsigned long long)sh->sector);
1010
1011         for (i = disks; i--; ) {
1012                 struct r5dev *dev = &sh->dev[i];
1013                 struct bio *chosen;
1014
1015                 if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) {
1016                         struct bio *wbi;
1017
1018                         spin_lock_irq(&sh->raid_conf->device_lock);
1019                         chosen = dev->towrite;
1020                         dev->towrite = NULL;
1021                         BUG_ON(dev->written);
1022                         wbi = dev->written = chosen;
1023                         spin_unlock_irq(&sh->raid_conf->device_lock);
1024
1025                         while (wbi && wbi->bi_sector <
1026                                 dev->sector + STRIPE_SECTORS) {
1027                                 if (wbi->bi_rw & REQ_FUA)
1028                                         set_bit(R5_WantFUA, &dev->flags);
1029                                 tx = async_copy_data(1, wbi, dev->page,
1030                                         dev->sector, tx);
1031                                 wbi = r5_next_bio(wbi, dev->sector);
1032                         }
1033                 }
1034         }
1035
1036         return tx;
1037 }
1038
1039 static void ops_complete_reconstruct(void *stripe_head_ref)
1040 {
1041         struct stripe_head *sh = stripe_head_ref;
1042         int disks = sh->disks;
1043         int pd_idx = sh->pd_idx;
1044         int qd_idx = sh->qd_idx;
1045         int i;
1046         bool fua = false;
1047
1048         pr_debug("%s: stripe %llu\n", __func__,
1049                 (unsigned long long)sh->sector);
1050
1051         for (i = disks; i--; )
1052                 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
1053
1054         for (i = disks; i--; ) {
1055                 struct r5dev *dev = &sh->dev[i];
1056
1057                 if (dev->written || i == pd_idx || i == qd_idx) {
1058                         set_bit(R5_UPTODATE, &dev->flags);
1059                         if (fua)
1060                                 set_bit(R5_WantFUA, &dev->flags);
1061                 }
1062         }
1063
1064         if (sh->reconstruct_state == reconstruct_state_drain_run)
1065                 sh->reconstruct_state = reconstruct_state_drain_result;
1066         else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
1067                 sh->reconstruct_state = reconstruct_state_prexor_drain_result;
1068         else {
1069                 BUG_ON(sh->reconstruct_state != reconstruct_state_run);
1070                 sh->reconstruct_state = reconstruct_state_result;
1071         }
1072
1073         set_bit(STRIPE_HANDLE, &sh->state);
1074         release_stripe(sh);
1075 }
1076
1077 static void
1078 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
1079                      struct dma_async_tx_descriptor *tx)
1080 {
1081         int disks = sh->disks;
1082         struct page **xor_srcs = percpu->scribble;
1083         struct async_submit_ctl submit;
1084         int count = 0, pd_idx = sh->pd_idx, i;
1085         struct page *xor_dest;
1086         int prexor = 0;
1087         unsigned long flags;
1088
1089         pr_debug("%s: stripe %llu\n", __func__,
1090                 (unsigned long long)sh->sector);
1091
1092         /* check if prexor is active which means only process blocks
1093          * that are part of a read-modify-write (written)
1094          */
1095         if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1096                 prexor = 1;
1097                 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1098                 for (i = disks; i--; ) {
1099                         struct r5dev *dev = &sh->dev[i];
1100                         if (dev->written)
1101                                 xor_srcs[count++] = dev->page;
1102                 }
1103         } else {
1104                 xor_dest = sh->dev[pd_idx].page;
1105                 for (i = disks; i--; ) {
1106                         struct r5dev *dev = &sh->dev[i];
1107                         if (i != pd_idx)
1108                                 xor_srcs[count++] = dev->page;
1109                 }
1110         }
1111
1112         /* 1/ if we prexor'd then the dest is reused as a source
1113          * 2/ if we did not prexor then we are redoing the parity
1114          * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
1115          * for the synchronous xor case
1116          */
1117         flags = ASYNC_TX_ACK |
1118                 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
1119
1120         atomic_inc(&sh->count);
1121
1122         init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh,
1123                           to_addr_conv(sh, percpu));
1124         if (unlikely(count == 1))
1125                 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1126         else
1127                 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1128 }
1129
1130 static void
1131 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
1132                      struct dma_async_tx_descriptor *tx)
1133 {
1134         struct async_submit_ctl submit;
1135         struct page **blocks = percpu->scribble;
1136         int count;
1137
1138         pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
1139
1140         count = set_syndrome_sources(blocks, sh);
1141
1142         atomic_inc(&sh->count);
1143
1144         init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct,
1145                           sh, to_addr_conv(sh, percpu));
1146         async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
1147 }
1148
1149 static void ops_complete_check(void *stripe_head_ref)
1150 {
1151         struct stripe_head *sh = stripe_head_ref;
1152
1153         pr_debug("%s: stripe %llu\n", __func__,
1154                 (unsigned long long)sh->sector);
1155
1156         sh->check_state = check_state_check_result;
1157         set_bit(STRIPE_HANDLE, &sh->state);
1158         release_stripe(sh);
1159 }
1160
1161 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
1162 {
1163         int disks = sh->disks;
1164         int pd_idx = sh->pd_idx;
1165         int qd_idx = sh->qd_idx;
1166         struct page *xor_dest;
1167         struct page **xor_srcs = percpu->scribble;
1168         struct dma_async_tx_descriptor *tx;
1169         struct async_submit_ctl submit;
1170         int count;
1171         int i;
1172
1173         pr_debug("%s: stripe %llu\n", __func__,
1174                 (unsigned long long)sh->sector);
1175
1176         count = 0;
1177         xor_dest = sh->dev[pd_idx].page;
1178         xor_srcs[count++] = xor_dest;
1179         for (i = disks; i--; ) {
1180                 if (i == pd_idx || i == qd_idx)
1181                         continue;
1182                 xor_srcs[count++] = sh->dev[i].page;
1183         }
1184
1185         init_async_submit(&submit, 0, NULL, NULL, NULL,
1186                           to_addr_conv(sh, percpu));
1187         tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
1188                            &sh->ops.zero_sum_result, &submit);
1189
1190         atomic_inc(&sh->count);
1191         init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
1192         tx = async_trigger_callback(&submit);
1193 }
1194
1195 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
1196 {
1197         struct page **srcs = percpu->scribble;
1198         struct async_submit_ctl submit;
1199         int count;
1200
1201         pr_debug("%s: stripe %llu checkp: %d\n", __func__,
1202                 (unsigned long long)sh->sector, checkp);
1203
1204         count = set_syndrome_sources(srcs, sh);
1205         if (!checkp)
1206                 srcs[count] = NULL;
1207
1208         atomic_inc(&sh->count);
1209         init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
1210                           sh, to_addr_conv(sh, percpu));
1211         async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
1212                            &sh->ops.zero_sum_result, percpu->spare_page, &submit);
1213 }
1214
1215 static void __raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1216 {
1217         int overlap_clear = 0, i, disks = sh->disks;
1218         struct dma_async_tx_descriptor *tx = NULL;
1219         raid5_conf_t *conf = sh->raid_conf;
1220         int level = conf->level;
1221         struct raid5_percpu *percpu;
1222         unsigned long cpu;
1223
1224         cpu = get_cpu();
1225         percpu = per_cpu_ptr(conf->percpu, cpu);
1226         if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
1227                 ops_run_biofill(sh);
1228                 overlap_clear++;
1229         }
1230
1231         if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
1232                 if (level < 6)
1233                         tx = ops_run_compute5(sh, percpu);
1234                 else {
1235                         if (sh->ops.target2 < 0 || sh->ops.target < 0)
1236                                 tx = ops_run_compute6_1(sh, percpu);
1237                         else
1238                                 tx = ops_run_compute6_2(sh, percpu);
1239                 }
1240                 /* terminate the chain if reconstruct is not set to be run */
1241                 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
1242                         async_tx_ack(tx);
1243         }
1244
1245         if (test_bit(STRIPE_OP_PREXOR, &ops_request))
1246                 tx = ops_run_prexor(sh, percpu, tx);
1247
1248         if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
1249                 tx = ops_run_biodrain(sh, tx);
1250                 overlap_clear++;
1251         }
1252
1253         if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
1254                 if (level < 6)
1255                         ops_run_reconstruct5(sh, percpu, tx);
1256                 else
1257                         ops_run_reconstruct6(sh, percpu, tx);
1258         }
1259
1260         if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
1261                 if (sh->check_state == check_state_run)
1262                         ops_run_check_p(sh, percpu);
1263                 else if (sh->check_state == check_state_run_q)
1264                         ops_run_check_pq(sh, percpu, 0);
1265                 else if (sh->check_state == check_state_run_pq)
1266                         ops_run_check_pq(sh, percpu, 1);
1267                 else
1268                         BUG();
1269         }
1270
1271         if (overlap_clear)
1272                 for (i = disks; i--; ) {
1273                         struct r5dev *dev = &sh->dev[i];
1274                         if (test_and_clear_bit(R5_Overlap, &dev->flags))
1275                                 wake_up(&sh->raid_conf->wait_for_overlap);
1276                 }
1277         put_cpu();
1278 }
1279
1280 #ifdef CONFIG_MULTICORE_RAID456
1281 static void async_run_ops(void *param, async_cookie_t cookie)
1282 {
1283         struct stripe_head *sh = param;
1284         unsigned long ops_request = sh->ops.request;
1285
1286         clear_bit_unlock(STRIPE_OPS_REQ_PENDING, &sh->state);
1287         wake_up(&sh->ops.wait_for_ops);
1288
1289         __raid_run_ops(sh, ops_request);
1290         release_stripe(sh);
1291 }
1292
1293 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1294 {
1295         /* since handle_stripe can be called outside of raid5d context
1296          * we need to ensure sh->ops.request is de-staged before another
1297          * request arrives
1298          */
1299         wait_event(sh->ops.wait_for_ops,
1300                    !test_and_set_bit_lock(STRIPE_OPS_REQ_PENDING, &sh->state));
1301         sh->ops.request = ops_request;
1302
1303         atomic_inc(&sh->count);
1304         async_schedule(async_run_ops, sh);
1305 }
1306 #else
1307 #define raid_run_ops __raid_run_ops
1308 #endif
1309
1310 static int grow_one_stripe(raid5_conf_t *conf)
1311 {
1312         struct stripe_head *sh;
1313         sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL);
1314         if (!sh)
1315                 return 0;
1316
1317         sh->raid_conf = conf;
1318         #ifdef CONFIG_MULTICORE_RAID456
1319         init_waitqueue_head(&sh->ops.wait_for_ops);
1320         #endif
1321
1322         if (grow_buffers(sh)) {
1323                 shrink_buffers(sh);
1324                 kmem_cache_free(conf->slab_cache, sh);
1325                 return 0;
1326         }
1327         /* we just created an active stripe so... */
1328         atomic_set(&sh->count, 1);
1329         atomic_inc(&conf->active_stripes);
1330         INIT_LIST_HEAD(&sh->lru);
1331         release_stripe(sh);
1332         return 1;
1333 }
1334
1335 static int grow_stripes(raid5_conf_t *conf, int num)
1336 {
1337         struct kmem_cache *sc;
1338         int devs = max(conf->raid_disks, conf->previous_raid_disks);
1339
1340         if (conf->mddev->gendisk)
1341                 sprintf(conf->cache_name[0],
1342                         "raid%d-%s", conf->level, mdname(conf->mddev));
1343         else
1344                 sprintf(conf->cache_name[0],
1345                         "raid%d-%p", conf->level, conf->mddev);
1346         sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);
1347
1348         conf->active_name = 0;
1349         sc = kmem_cache_create(conf->cache_name[conf->active_name],
1350                                sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
1351                                0, 0, NULL);
1352         if (!sc)
1353                 return 1;
1354         conf->slab_cache = sc;
1355         conf->pool_size = devs;
1356         while (num--)
1357                 if (!grow_one_stripe(conf))
1358                         return 1;
1359         return 0;
1360 }
1361
1362 /**
1363  * scribble_len - return the required size of the scribble region
1364  * @num - total number of disks in the array
1365  *
1366  * The size must be enough to contain:
1367  * 1/ a struct page pointer for each device in the array +2
1368  * 2/ room to convert each entry in (1) to its corresponding dma
1369  *    (dma_map_page()) or page (page_address()) address.
1370  *
1371  * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
1372  * calculate over all devices (not just the data blocks), using zeros in place
1373  * of the P and Q blocks.
1374  */
1375 static size_t scribble_len(int num)
1376 {
1377         size_t len;
1378
1379         len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
1380
1381         return len;
1382 }
1383
1384 static int resize_stripes(raid5_conf_t *conf, int newsize)
1385 {
1386         /* Make all the stripes able to hold 'newsize' devices.
1387          * New slots in each stripe get 'page' set to a new page.
1388          *
1389          * This happens in stages:
1390          * 1/ create a new kmem_cache and allocate the required number of
1391          *    stripe_heads.
1392          * 2/ gather all the old stripe_heads and tranfer the pages across
1393          *    to the new stripe_heads.  This will have the side effect of
1394          *    freezing the array as once all stripe_heads have been collected,
1395          *    no IO will be possible.  Old stripe heads are freed once their
1396          *    pages have been transferred over, and the old kmem_cache is
1397          *    freed when all stripes are done.
1398          * 3/ reallocate conf->disks to be suitable bigger.  If this fails,
1399          *    we simple return a failre status - no need to clean anything up.
1400          * 4/ allocate new pages for the new slots in the new stripe_heads.
1401          *    If this fails, we don't bother trying the shrink the
1402          *    stripe_heads down again, we just leave them as they are.
1403          *    As each stripe_head is processed the new one is released into
1404          *    active service.
1405          *
1406          * Once step2 is started, we cannot afford to wait for a write,
1407          * so we use GFP_NOIO allocations.
1408          */
1409         struct stripe_head *osh, *nsh;
1410         LIST_HEAD(newstripes);
1411         struct disk_info *ndisks;
1412         unsigned long cpu;
1413         int err;
1414         struct kmem_cache *sc;
1415         int i;
1416
1417         if (newsize <= conf->pool_size)
1418                 return 0; /* never bother to shrink */
1419
1420         err = md_allow_write(conf->mddev);
1421         if (err)
1422                 return err;
1423
1424         /* Step 1 */
1425         sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
1426                                sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
1427                                0, 0, NULL);
1428         if (!sc)
1429                 return -ENOMEM;
1430
1431         for (i = conf->max_nr_stripes; i; i--) {
1432                 nsh = kmem_cache_zalloc(sc, GFP_KERNEL);
1433                 if (!nsh)
1434                         break;
1435
1436                 nsh->raid_conf = conf;
1437                 #ifdef CONFIG_MULTICORE_RAID456
1438                 init_waitqueue_head(&nsh->ops.wait_for_ops);
1439                 #endif
1440
1441                 list_add(&nsh->lru, &newstripes);
1442         }
1443         if (i) {
1444                 /* didn't get enough, give up */
1445                 while (!list_empty(&newstripes)) {
1446                         nsh = list_entry(newstripes.next, struct stripe_head, lru);
1447                         list_del(&nsh->lru);
1448                         kmem_cache_free(sc, nsh);
1449                 }
1450                 kmem_cache_destroy(sc);
1451                 return -ENOMEM;
1452         }
1453         /* Step 2 - Must use GFP_NOIO now.
1454          * OK, we have enough stripes, start collecting inactive
1455          * stripes and copying them over
1456          */
1457         list_for_each_entry(nsh, &newstripes, lru) {
1458                 spin_lock_irq(&conf->device_lock);
1459                 wait_event_lock_irq(conf->wait_for_stripe,
1460                                     !list_empty(&conf->inactive_list),
1461                                     conf->device_lock,
1462                                     );
1463                 osh = get_free_stripe(conf);
1464                 spin_unlock_irq(&conf->device_lock);
1465                 atomic_set(&nsh->count, 1);
1466                 for(i=0; i<conf->pool_size; i++)
1467                         nsh->dev[i].page = osh->dev[i].page;
1468                 for( ; i<newsize; i++)
1469                         nsh->dev[i].page = NULL;
1470                 kmem_cache_free(conf->slab_cache, osh);
1471         }
1472         kmem_cache_destroy(conf->slab_cache);
1473
1474         /* Step 3.
1475          * At this point, we are holding all the stripes so the array
1476          * is completely stalled, so now is a good time to resize
1477          * conf->disks and the scribble region
1478          */
1479         ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
1480         if (ndisks) {
1481                 for (i=0; i<conf->raid_disks; i++)
1482                         ndisks[i] = conf->disks[i];
1483                 kfree(conf->disks);
1484                 conf->disks = ndisks;
1485         } else
1486                 err = -ENOMEM;
1487
1488         get_online_cpus();
1489         conf->scribble_len = scribble_len(newsize);
1490         for_each_present_cpu(cpu) {
1491                 struct raid5_percpu *percpu;
1492                 void *scribble;
1493
1494                 percpu = per_cpu_ptr(conf->percpu, cpu);
1495                 scribble = kmalloc(conf->scribble_len, GFP_NOIO);
1496
1497                 if (scribble) {
1498                         kfree(percpu->scribble);
1499                         percpu->scribble = scribble;
1500                 } else {
1501                         err = -ENOMEM;
1502                         break;
1503                 }
1504         }
1505         put_online_cpus();
1506
1507         /* Step 4, return new stripes to service */
1508         while(!list_empty(&newstripes)) {
1509                 nsh = list_entry(newstripes.next, struct stripe_head, lru);
1510                 list_del_init(&nsh->lru);
1511
1512                 for (i=conf->raid_disks; i < newsize; i++)
1513                         if (nsh->dev[i].page == NULL) {
1514                                 struct page *p = alloc_page(GFP_NOIO);
1515                                 nsh->dev[i].page = p;
1516                                 if (!p)
1517                                         err = -ENOMEM;
1518                         }
1519                 release_stripe(nsh);
1520         }
1521         /* critical section pass, GFP_NOIO no longer needed */
1522
1523         conf->slab_cache = sc;
1524         conf->active_name = 1-conf->active_name;
1525         conf->pool_size = newsize;
1526         return err;
1527 }
1528
1529 static int drop_one_stripe(raid5_conf_t *conf)
1530 {
1531         struct stripe_head *sh;
1532
1533         spin_lock_irq(&conf->device_lock);
1534         sh = get_free_stripe(conf);
1535         spin_unlock_irq(&conf->device_lock);
1536         if (!sh)
1537                 return 0;
1538         BUG_ON(atomic_read(&sh->count));
1539         shrink_buffers(sh);
1540         kmem_cache_free(conf->slab_cache, sh);
1541         atomic_dec(&conf->active_stripes);
1542         return 1;
1543 }
1544
1545 static void shrink_stripes(raid5_conf_t *conf)
1546 {
1547         while (drop_one_stripe(conf))
1548                 ;
1549
1550         if (conf->slab_cache)
1551                 kmem_cache_destroy(conf->slab_cache);
1552         conf->slab_cache = NULL;
1553 }
1554
1555 static void raid5_end_read_request(struct bio * bi, int error)
1556 {
1557         struct stripe_head *sh = bi->bi_private;
1558         raid5_conf_t *conf = sh->raid_conf;
1559         int disks = sh->disks, i;
1560         int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1561         char b[BDEVNAME_SIZE];
1562         mdk_rdev_t *rdev;
1563
1564
1565         for (i=0 ; i<disks; i++)
1566                 if (bi == &sh->dev[i].req)
1567                         break;
1568
1569         pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
1570                 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
1571                 uptodate);
1572         if (i == disks) {
1573                 BUG();
1574                 return;
1575         }
1576
1577         if (uptodate) {
1578                 set_bit(R5_UPTODATE, &sh->dev[i].flags);
1579                 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
1580                         rdev = conf->disks[i].rdev;
1581                         printk_ratelimited(
1582                                 KERN_INFO
1583                                 "md/raid:%s: read error corrected"
1584                                 " (%lu sectors at %llu on %s)\n",
1585                                 mdname(conf->mddev), STRIPE_SECTORS,
1586                                 (unsigned long long)(sh->sector
1587                                                      + rdev->data_offset),
1588                                 bdevname(rdev->bdev, b));
1589                         atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
1590                         clear_bit(R5_ReadError, &sh->dev[i].flags);
1591                         clear_bit(R5_ReWrite, &sh->dev[i].flags);
1592                 }
1593                 if (atomic_read(&conf->disks[i].rdev->read_errors))
1594                         atomic_set(&conf->disks[i].rdev->read_errors, 0);
1595         } else {
1596                 const char *bdn = bdevname(conf->disks[i].rdev->bdev, b);
1597                 int retry = 0;
1598                 rdev = conf->disks[i].rdev;
1599
1600                 clear_bit(R5_UPTODATE, &sh->dev[i].flags);
1601                 atomic_inc(&rdev->read_errors);
1602                 if (conf->mddev->degraded >= conf->max_degraded)
1603                         printk_ratelimited(
1604                                 KERN_WARNING
1605                                 "md/raid:%s: read error not correctable "
1606                                 "(sector %llu on %s).\n",
1607                                 mdname(conf->mddev),
1608                                 (unsigned long long)(sh->sector
1609                                                      + rdev->data_offset),
1610                                 bdn);
1611                 else if (test_bit(R5_ReWrite, &sh->dev[i].flags))
1612                         /* Oh, no!!! */
1613                         printk_ratelimited(
1614                                 KERN_WARNING
1615                                 "md/raid:%s: read error NOT corrected!! "
1616                                 "(sector %llu on %s).\n",
1617                                 mdname(conf->mddev),
1618                                 (unsigned long long)(sh->sector
1619                                                      + rdev->data_offset),
1620                                 bdn);
1621                 else if (atomic_read(&rdev->read_errors)
1622                          > conf->max_nr_stripes)
1623                         printk(KERN_WARNING
1624                                "md/raid:%s: Too many read errors, failing device %s.\n",
1625                                mdname(conf->mddev), bdn);
1626                 else
1627                         retry = 1;
1628                 if (retry)
1629                         set_bit(R5_ReadError, &sh->dev[i].flags);
1630                 else {
1631                         clear_bit(R5_ReadError, &sh->dev[i].flags);
1632                         clear_bit(R5_ReWrite, &sh->dev[i].flags);
1633                         md_error(conf->mddev, rdev);
1634                 }
1635         }
1636         rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
1637         clear_bit(R5_LOCKED, &sh->dev[i].flags);
1638         set_bit(STRIPE_HANDLE, &sh->state);
1639         release_stripe(sh);
1640 }
1641
1642 static void raid5_end_write_request(struct bio *bi, int error)
1643 {
1644         struct stripe_head *sh = bi->bi_private;
1645         raid5_conf_t *conf = sh->raid_conf;
1646         int disks = sh->disks, i;
1647         int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1648
1649         for (i=0 ; i<disks; i++)
1650                 if (bi == &sh->dev[i].req)
1651                         break;
1652
1653         pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
1654                 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
1655                 uptodate);
1656         if (i == disks) {
1657                 BUG();
1658                 return;
1659         }
1660
1661         if (!uptodate)
1662                 md_error(conf->mddev, conf->disks[i].rdev);
1663
1664         rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
1665         
1666         clear_bit(R5_LOCKED, &sh->dev[i].flags);
1667         set_bit(STRIPE_HANDLE, &sh->state);
1668         release_stripe(sh);
1669 }
1670
1671
1672 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
1673         
1674 static void raid5_build_block(struct stripe_head *sh, int i, int previous)
1675 {
1676         struct r5dev *dev = &sh->dev[i];
1677
1678         bio_init(&dev->req);
1679         dev->req.bi_io_vec = &dev->vec;
1680         dev->req.bi_vcnt++;
1681         dev->req.bi_max_vecs++;
1682         dev->vec.bv_page = dev->page;
1683         dev->vec.bv_len = STRIPE_SIZE;
1684         dev->vec.bv_offset = 0;
1685
1686         dev->req.bi_sector = sh->sector;
1687         dev->req.bi_private = sh;
1688
1689         dev->flags = 0;
1690         dev->sector = compute_blocknr(sh, i, previous);
1691 }
1692
1693 static void error(mddev_t *mddev, mdk_rdev_t *rdev)
1694 {
1695         char b[BDEVNAME_SIZE];
1696         raid5_conf_t *conf = mddev->private;
1697         pr_debug("raid456: error called\n");
1698
1699         if (test_and_clear_bit(In_sync, &rdev->flags)) {
1700                 unsigned long flags;
1701                 spin_lock_irqsave(&conf->device_lock, flags);
1702                 mddev->degraded++;
1703                 spin_unlock_irqrestore(&conf->device_lock, flags);
1704                 /*
1705                  * if recovery was running, make sure it aborts.
1706                  */
1707                 set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1708         }
1709         set_bit(Faulty, &rdev->flags);
1710         set_bit(MD_CHANGE_DEVS, &mddev->flags);
1711         printk(KERN_ALERT
1712                "md/raid:%s: Disk failure on %s, disabling device.\n"
1713                "md/raid:%s: Operation continuing on %d devices.\n",
1714                mdname(mddev),
1715                bdevname(rdev->bdev, b),
1716                mdname(mddev),
1717                conf->raid_disks - mddev->degraded);
1718 }
1719
1720 /*
1721  * Input: a 'big' sector number,
1722  * Output: index of the data and parity disk, and the sector # in them.
1723  */
1724 static sector_t raid5_compute_sector(raid5_conf_t *conf, sector_t r_sector,
1725                                      int previous, int *dd_idx,
1726                                      struct stripe_head *sh)
1727 {
1728         sector_t stripe, stripe2;
1729         sector_t chunk_number;
1730         unsigned int chunk_offset;
1731         int pd_idx, qd_idx;
1732         int ddf_layout = 0;
1733         sector_t new_sector;
1734         int algorithm = previous ? conf->prev_algo
1735                                  : conf->algorithm;
1736         int sectors_per_chunk = previous ? conf->prev_chunk_sectors
1737                                          : conf->chunk_sectors;
1738         int raid_disks = previous ? conf->previous_raid_disks
1739                                   : conf->raid_disks;
1740         int data_disks = raid_disks - conf->max_degraded;
1741
1742         /* First compute the information on this sector */
1743
1744         /*
1745          * Compute the chunk number and the sector offset inside the chunk
1746          */
1747         chunk_offset = sector_div(r_sector, sectors_per_chunk);
1748         chunk_number = r_sector;
1749
1750         /*
1751          * Compute the stripe number
1752          */
1753         stripe = chunk_number;
1754         *dd_idx = sector_div(stripe, data_disks);
1755         stripe2 = stripe;
1756         /*
1757          * Select the parity disk based on the user selected algorithm.
1758          */
1759         pd_idx = qd_idx = -1;
1760         switch(conf->level) {
1761         case 4:
1762                 pd_idx = data_disks;
1763                 break;
1764         case 5:
1765                 switch (algorithm) {
1766                 case ALGORITHM_LEFT_ASYMMETRIC:
1767                         pd_idx = data_disks - sector_div(stripe2, raid_disks);
1768                         if (*dd_idx >= pd_idx)
1769                                 (*dd_idx)++;
1770                         break;
1771                 case ALGORITHM_RIGHT_ASYMMETRIC:
1772                         pd_idx = sector_div(stripe2, raid_disks);
1773                         if (*dd_idx >= pd_idx)
1774                                 (*dd_idx)++;
1775                         break;
1776                 case ALGORITHM_LEFT_SYMMETRIC:
1777                         pd_idx = data_disks - sector_div(stripe2, raid_disks);
1778                         *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1779                         break;
1780                 case ALGORITHM_RIGHT_SYMMETRIC:
1781                         pd_idx = sector_div(stripe2, raid_disks);
1782                         *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1783                         break;
1784                 case ALGORITHM_PARITY_0:
1785                         pd_idx = 0;
1786                         (*dd_idx)++;
1787                         break;
1788                 case ALGORITHM_PARITY_N:
1789                         pd_idx = data_disks;
1790                         break;
1791                 default:
1792                         BUG();
1793                 }
1794                 break;
1795         case 6:
1796
1797                 switch (algorithm) {
1798                 case ALGORITHM_LEFT_ASYMMETRIC:
1799                         pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
1800                         qd_idx = pd_idx + 1;
1801                         if (pd_idx == raid_disks-1) {
1802                                 (*dd_idx)++;    /* Q D D D P */
1803                                 qd_idx = 0;
1804                         } else if (*dd_idx >= pd_idx)
1805                                 (*dd_idx) += 2; /* D D P Q D */
1806                         break;
1807                 case ALGORITHM_RIGHT_ASYMMETRIC:
1808                         pd_idx = sector_div(stripe2, raid_disks);
1809                         qd_idx = pd_idx + 1;
1810                         if (pd_idx == raid_disks-1) {
1811                                 (*dd_idx)++;    /* Q D D D P */
1812                                 qd_idx = 0;
1813                         } else if (*dd_idx >= pd_idx)
1814                                 (*dd_idx) += 2; /* D D P Q D */
1815                         break;
1816                 case ALGORITHM_LEFT_SYMMETRIC:
1817                         pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
1818                         qd_idx = (pd_idx + 1) % raid_disks;
1819                         *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
1820                         break;
1821                 case ALGORITHM_RIGHT_SYMMETRIC:
1822                         pd_idx = sector_div(stripe2, raid_disks);
1823                         qd_idx = (pd_idx + 1) % raid_disks;
1824                         *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
1825                         break;
1826
1827                 case ALGORITHM_PARITY_0:
1828                         pd_idx = 0;
1829                         qd_idx = 1;
1830                         (*dd_idx) += 2;
1831                         break;
1832                 case ALGORITHM_PARITY_N:
1833                         pd_idx = data_disks;
1834                         qd_idx = data_disks + 1;
1835                         break;
1836
1837                 case ALGORITHM_ROTATING_ZERO_RESTART:
1838                         /* Exactly the same as RIGHT_ASYMMETRIC, but or
1839                          * of blocks for computing Q is different.
1840                          */
1841                         pd_idx = sector_div(stripe2, raid_disks);
1842                         qd_idx = pd_idx + 1;
1843                         if (pd_idx == raid_disks-1) {
1844                                 (*dd_idx)++;    /* Q D D D P */
1845                                 qd_idx = 0;
1846                         } else if (*dd_idx >= pd_idx)
1847                                 (*dd_idx) += 2; /* D D P Q D */
1848                         ddf_layout = 1;
1849                         break;
1850
1851                 case ALGORITHM_ROTATING_N_RESTART:
1852                         /* Same a left_asymmetric, by first stripe is
1853                          * D D D P Q  rather than
1854                          * Q D D D P
1855                          */
1856                         stripe2 += 1;
1857                         pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
1858                         qd_idx = pd_idx + 1;
1859                         if (pd_idx == raid_disks-1) {
1860                                 (*dd_idx)++;    /* Q D D D P */
1861                                 qd_idx = 0;
1862                         } else if (*dd_idx >= pd_idx)
1863                                 (*dd_idx) += 2; /* D D P Q D */
1864                         ddf_layout = 1;
1865                         break;
1866
1867                 case ALGORITHM_ROTATING_N_CONTINUE:
1868                         /* Same as left_symmetric but Q is before P */
1869                         pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
1870                         qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
1871                         *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1872                         ddf_layout = 1;
1873                         break;
1874
1875                 case ALGORITHM_LEFT_ASYMMETRIC_6:
1876                         /* RAID5 left_asymmetric, with Q on last device */
1877                         pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
1878                         if (*dd_idx >= pd_idx)
1879                                 (*dd_idx)++;
1880                         qd_idx = raid_disks - 1;
1881                         break;
1882
1883                 case ALGORITHM_RIGHT_ASYMMETRIC_6:
1884                         pd_idx = sector_div(stripe2, raid_disks-1);
1885                         if (*dd_idx >= pd_idx)
1886                                 (*dd_idx)++;
1887                         qd_idx = raid_disks - 1;
1888                         break;
1889
1890                 case ALGORITHM_LEFT_SYMMETRIC_6:
1891                         pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
1892                         *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
1893                         qd_idx = raid_disks - 1;
1894                         break;
1895
1896                 case ALGORITHM_RIGHT_SYMMETRIC_6:
1897                         pd_idx = sector_div(stripe2, raid_disks-1);
1898                         *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
1899                         qd_idx = raid_disks - 1;
1900                         break;
1901
1902                 case ALGORITHM_PARITY_0_6:
1903                         pd_idx = 0;
1904                         (*dd_idx)++;
1905                         qd_idx = raid_disks - 1;
1906                         break;
1907
1908                 default:
1909                         BUG();
1910                 }
1911                 break;
1912         }
1913
1914         if (sh) {
1915                 sh->pd_idx = pd_idx;
1916                 sh->qd_idx = qd_idx;
1917                 sh->ddf_layout = ddf_layout;
1918         }
1919         /*
1920          * Finally, compute the new sector number
1921          */
1922         new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
1923         return new_sector;
1924 }
1925
1926
1927 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
1928 {
1929         raid5_conf_t *conf = sh->raid_conf;
1930         int raid_disks = sh->disks;
1931         int data_disks = raid_disks - conf->max_degraded;
1932         sector_t new_sector = sh->sector, check;
1933         int sectors_per_chunk = previous ? conf->prev_chunk_sectors
1934                                          : conf->chunk_sectors;
1935         int algorithm = previous ? conf->prev_algo
1936                                  : conf->algorithm;
1937         sector_t stripe;
1938         int chunk_offset;
1939         sector_t chunk_number;
1940         int dummy1, dd_idx = i;
1941         sector_t r_sector;
1942         struct stripe_head sh2;
1943
1944
1945         chunk_offset = sector_div(new_sector, sectors_per_chunk);
1946         stripe = new_sector;
1947
1948         if (i == sh->pd_idx)
1949                 return 0;
1950         switch(conf->level) {
1951         case 4: break;
1952         case 5:
1953                 switch (algorithm) {
1954                 case ALGORITHM_LEFT_ASYMMETRIC:
1955                 case ALGORITHM_RIGHT_ASYMMETRIC:
1956                         if (i > sh->pd_idx)
1957                                 i--;
1958                         break;
1959                 case ALGORITHM_LEFT_SYMMETRIC:
1960                 case ALGORITHM_RIGHT_SYMMETRIC:
1961                         if (i < sh->pd_idx)
1962                                 i += raid_disks;
1963                         i -= (sh->pd_idx + 1);
1964                         break;
1965                 case ALGORITHM_PARITY_0:
1966                         i -= 1;
1967                         break;
1968                 case ALGORITHM_PARITY_N:
1969                         break;
1970                 default:
1971                         BUG();
1972                 }
1973                 break;
1974         case 6:
1975                 if (i == sh->qd_idx)
1976                         return 0; /* It is the Q disk */
1977                 switch (algorithm) {
1978                 case ALGORITHM_LEFT_ASYMMETRIC:
1979                 case ALGORITHM_RIGHT_ASYMMETRIC:
1980                 case ALGORITHM_ROTATING_ZERO_RESTART:
1981                 case ALGORITHM_ROTATING_N_RESTART:
1982                         if (sh->pd_idx == raid_disks-1)
1983                                 i--;    /* Q D D D P */
1984                         else if (i > sh->pd_idx)
1985                                 i -= 2; /* D D P Q D */
1986                         break;
1987                 case ALGORITHM_LEFT_SYMMETRIC:
1988                 case ALGORITHM_RIGHT_SYMMETRIC:
1989                         if (sh->pd_idx == raid_disks-1)
1990                                 i--; /* Q D D D P */
1991                         else {
1992                                 /* D D P Q D */
1993                                 if (i < sh->pd_idx)
1994                                         i += raid_disks;
1995                                 i -= (sh->pd_idx + 2);
1996                         }
1997                         break;
1998                 case ALGORITHM_PARITY_0:
1999                         i -= 2;
2000                         break;
2001                 case ALGORITHM_PARITY_N:
2002                         break;
2003                 case ALGORITHM_ROTATING_N_CONTINUE:
2004                         /* Like left_symmetric, but P is before Q */
2005                         if (sh->pd_idx == 0)
2006                                 i--;    /* P D D D Q */
2007                         else {
2008                                 /* D D Q P D */
2009                                 if (i < sh->pd_idx)
2010                                         i += raid_disks;
2011                                 i -= (sh->pd_idx + 1);
2012                         }
2013                         break;
2014                 case ALGORITHM_LEFT_ASYMMETRIC_6:
2015                 case ALGORITHM_RIGHT_ASYMMETRIC_6:
2016                         if (i > sh->pd_idx)
2017                                 i--;
2018                         break;
2019                 case ALGORITHM_LEFT_SYMMETRIC_6:
2020                 case ALGORITHM_RIGHT_SYMMETRIC_6:
2021                         if (i < sh->pd_idx)
2022                                 i += data_disks + 1;
2023                         i -= (sh->pd_idx + 1);
2024                         break;
2025                 case ALGORITHM_PARITY_0_6:
2026                         i -= 1;
2027                         break;
2028                 default:
2029                         BUG();
2030                 }
2031                 break;
2032         }
2033
2034         chunk_number = stripe * data_disks + i;
2035         r_sector = chunk_number * sectors_per_chunk + chunk_offset;
2036
2037         check = raid5_compute_sector(conf, r_sector,
2038                                      previous, &dummy1, &sh2);
2039         if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
2040                 || sh2.qd_idx != sh->qd_idx) {
2041                 printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
2042                        mdname(conf->mddev));
2043                 return 0;
2044         }
2045         return r_sector;
2046 }
2047
2048
2049 static void
2050 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
2051                          int rcw, int expand)
2052 {
2053         int i, pd_idx = sh->pd_idx, disks = sh->disks;
2054         raid5_conf_t *conf = sh->raid_conf;
2055         int level = conf->level;
2056
2057         if (rcw) {
2058                 /* if we are not expanding this is a proper write request, and
2059                  * there will be bios with new data to be drained into the
2060                  * stripe cache
2061                  */
2062                 if (!expand) {
2063                         sh->reconstruct_state = reconstruct_state_drain_run;
2064                         set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2065                 } else
2066                         sh->reconstruct_state = reconstruct_state_run;
2067
2068                 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2069
2070                 for (i = disks; i--; ) {
2071                         struct r5dev *dev = &sh->dev[i];
2072
2073                         if (dev->towrite) {
2074                                 set_bit(R5_LOCKED, &dev->flags);
2075                                 set_bit(R5_Wantdrain, &dev->flags);
2076                                 if (!expand)
2077                                         clear_bit(R5_UPTODATE, &dev->flags);
2078                                 s->locked++;
2079                         }
2080                 }
2081                 if (s->locked + conf->max_degraded == disks)
2082                         if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
2083                                 atomic_inc(&conf->pending_full_writes);
2084         } else {
2085                 BUG_ON(level == 6);
2086                 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
2087                         test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
2088
2089                 sh->reconstruct_state = reconstruct_state_prexor_drain_run;
2090                 set_bit(STRIPE_OP_PREXOR, &s->ops_request);
2091                 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2092                 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2093
2094                 for (i = disks; i--; ) {
2095                         struct r5dev *dev = &sh->dev[i];
2096                         if (i == pd_idx)
2097                                 continue;
2098
2099                         if (dev->towrite &&
2100                             (test_bit(R5_UPTODATE, &dev->flags) ||
2101                              test_bit(R5_Wantcompute, &dev->flags))) {
2102                                 set_bit(R5_Wantdrain, &dev->flags);
2103                                 set_bit(R5_LOCKED, &dev->flags);
2104                                 clear_bit(R5_UPTODATE, &dev->flags);
2105                                 s->locked++;
2106                         }
2107                 }
2108         }
2109
2110         /* keep the parity disk(s) locked while asynchronous operations
2111          * are in flight
2112          */
2113         set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
2114         clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2115         s->locked++;
2116
2117         if (level == 6) {
2118                 int qd_idx = sh->qd_idx;
2119                 struct r5dev *dev = &sh->dev[qd_idx];
2120
2121                 set_bit(R5_LOCKED, &dev->flags);
2122                 clear_bit(R5_UPTODATE, &dev->flags);
2123                 s->locked++;
2124         }
2125
2126         pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
2127                 __func__, (unsigned long long)sh->sector,
2128                 s->locked, s->ops_request);
2129 }
2130
2131 /*
2132  * Each stripe/dev can have one or more bion attached.
2133  * toread/towrite point to the first in a chain.
2134  * The bi_next chain must be in order.
2135  */
2136 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite)
2137 {
2138         struct bio **bip;
2139         raid5_conf_t *conf = sh->raid_conf;
2140         int firstwrite=0;
2141
2142         pr_debug("adding bi b#%llu to stripe s#%llu\n",
2143                 (unsigned long long)bi->bi_sector,
2144                 (unsigned long long)sh->sector);
2145
2146
2147         spin_lock_irq(&conf->device_lock);
2148         if (forwrite) {
2149                 bip = &sh->dev[dd_idx].towrite;
2150                 if (*bip == NULL && sh->dev[dd_idx].written == NULL)
2151                         firstwrite = 1;
2152         } else
2153                 bip = &sh->dev[dd_idx].toread;
2154         while (*bip && (*bip)->bi_sector < bi->bi_sector) {
2155                 if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector)
2156                         goto overlap;
2157                 bip = & (*bip)->bi_next;
2158         }
2159         if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9))
2160                 goto overlap;
2161
2162         BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
2163         if (*bip)
2164                 bi->bi_next = *bip;
2165         *bip = bi;
2166         bi->bi_phys_segments++;
2167
2168         if (forwrite) {
2169                 /* check if page is covered */
2170                 sector_t sector = sh->dev[dd_idx].sector;
2171                 for (bi=sh->dev[dd_idx].towrite;
2172                      sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
2173                              bi && bi->bi_sector <= sector;
2174                      bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
2175                         if (bi->bi_sector + (bi->bi_size>>9) >= sector)
2176                                 sector = bi->bi_sector + (bi->bi_size>>9);
2177                 }
2178                 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
2179                         set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags);
2180         }
2181         spin_unlock_irq(&conf->device_lock);
2182
2183         pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
2184                 (unsigned long long)(*bip)->bi_sector,
2185                 (unsigned long long)sh->sector, dd_idx);
2186
2187         if (conf->mddev->bitmap && firstwrite) {
2188                 bitmap_startwrite(conf->mddev->bitmap, sh->sector,
2189                                   STRIPE_SECTORS, 0);
2190                 sh->bm_seq = conf->seq_flush+1;
2191                 set_bit(STRIPE_BIT_DELAY, &sh->state);
2192         }
2193         return 1;
2194
2195  overlap:
2196         set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
2197         spin_unlock_irq(&conf->device_lock);
2198         return 0;
2199 }
2200
2201 static void end_reshape(raid5_conf_t *conf);
2202
2203 static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous,
2204                             struct stripe_head *sh)
2205 {
2206         int sectors_per_chunk =
2207                 previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
2208         int dd_idx;
2209         int chunk_offset = sector_div(stripe, sectors_per_chunk);
2210         int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
2211
2212         raid5_compute_sector(conf,
2213                              stripe * (disks - conf->max_degraded)
2214                              *sectors_per_chunk + chunk_offset,
2215                              previous,
2216                              &dd_idx, sh);
2217 }
2218
2219 static void
2220 handle_failed_stripe(raid5_conf_t *conf, struct stripe_head *sh,
2221                                 struct stripe_head_state *s, int disks,
2222                                 struct bio **return_bi)
2223 {
2224         int i;
2225         for (i = disks; i--; ) {
2226                 struct bio *bi;
2227                 int bitmap_end = 0;
2228
2229                 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2230                         mdk_rdev_t *rdev;
2231                         rcu_read_lock();
2232                         rdev = rcu_dereference(conf->disks[i].rdev);
2233                         if (rdev && test_bit(In_sync, &rdev->flags))
2234                                 /* multiple read failures in one stripe */
2235                                 md_error(conf->mddev, rdev);
2236                         rcu_read_unlock();
2237                 }
2238                 spin_lock_irq(&conf->device_lock);
2239                 /* fail all writes first */
2240                 bi = sh->dev[i].towrite;
2241                 sh->dev[i].towrite = NULL;
2242                 if (bi) {
2243                         s->to_write--;
2244                         bitmap_end = 1;
2245                 }
2246
2247                 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2248                         wake_up(&conf->wait_for_overlap);
2249
2250                 while (bi && bi->bi_sector <
2251                         sh->dev[i].sector + STRIPE_SECTORS) {
2252                         struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
2253                         clear_bit(BIO_UPTODATE, &bi->bi_flags);
2254                         if (!raid5_dec_bi_phys_segments(bi)) {
2255                                 md_write_end(conf->mddev);
2256                                 bi->bi_next = *return_bi;
2257                                 *return_bi = bi;
2258                         }
2259                         bi = nextbi;
2260                 }
2261                 /* and fail all 'written' */
2262                 bi = sh->dev[i].written;
2263                 sh->dev[i].written = NULL;
2264                 if (bi) bitmap_end = 1;
2265                 while (bi && bi->bi_sector <
2266                        sh->dev[i].sector + STRIPE_SECTORS) {
2267                         struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
2268                         clear_bit(BIO_UPTODATE, &bi->bi_flags);
2269                         if (!raid5_dec_bi_phys_segments(bi)) {
2270                                 md_write_end(conf->mddev);
2271                                 bi->bi_next = *return_bi;
2272                                 *return_bi = bi;
2273                         }
2274                         bi = bi2;
2275                 }
2276
2277                 /* fail any reads if this device is non-operational and
2278                  * the data has not reached the cache yet.
2279                  */
2280                 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
2281                     (!test_bit(R5_Insync, &sh->dev[i].flags) ||
2282                       test_bit(R5_ReadError, &sh->dev[i].flags))) {
2283                         bi = sh->dev[i].toread;
2284                         sh->dev[i].toread = NULL;
2285                         if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2286                                 wake_up(&conf->wait_for_overlap);
2287                         if (bi) s->to_read--;
2288                         while (bi && bi->bi_sector <
2289                                sh->dev[i].sector + STRIPE_SECTORS) {
2290                                 struct bio *nextbi =
2291                                         r5_next_bio(bi, sh->dev[i].sector);
2292                                 clear_bit(BIO_UPTODATE, &bi->bi_flags);
2293                                 if (!raid5_dec_bi_phys_segments(bi)) {
2294                                         bi->bi_next = *return_bi;
2295                                         *return_bi = bi;
2296                                 }
2297                                 bi = nextbi;
2298                         }
2299                 }
2300                 spin_unlock_irq(&conf->device_lock);
2301                 if (bitmap_end)
2302                         bitmap_endwrite(conf->mddev->bitmap, sh->sector,
2303                                         STRIPE_SECTORS, 0, 0);
2304         }
2305
2306         if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2307                 if (atomic_dec_and_test(&conf->pending_full_writes))
2308                         md_wakeup_thread(conf->mddev->thread);
2309 }
2310
2311 /* fetch_block - checks the given member device to see if its data needs
2312  * to be read or computed to satisfy a request.
2313  *
2314  * Returns 1 when no more member devices need to be checked, otherwise returns
2315  * 0 to tell the loop in handle_stripe_fill to continue
2316  */
2317 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
2318                        int disk_idx, int disks)
2319 {
2320         struct r5dev *dev = &sh->dev[disk_idx];
2321         struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
2322                                   &sh->dev[s->failed_num[1]] };
2323
2324         /* is the data in this block needed, and can we get it? */
2325         if (!test_bit(R5_LOCKED, &dev->flags) &&
2326             !test_bit(R5_UPTODATE, &dev->flags) &&
2327             (dev->toread ||
2328              (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
2329              s->syncing || s->expanding ||
2330              (s->failed >= 1 && fdev[0]->toread) ||
2331              (s->failed >= 2 && fdev[1]->toread) ||
2332              (sh->raid_conf->level <= 5 && s->failed && fdev[0]->towrite &&
2333               !test_bit(R5_OVERWRITE, &fdev[0]->flags)) ||
2334              (sh->raid_conf->level == 6 && s->failed && s->to_write))) {
2335                 /* we would like to get this block, possibly by computing it,
2336                  * otherwise read it if the backing disk is insync
2337                  */
2338                 BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
2339                 BUG_ON(test_bit(R5_Wantread, &dev->flags));
2340                 if ((s->uptodate == disks - 1) &&
2341                     (s->failed && (disk_idx == s->failed_num[0] ||
2342                                    disk_idx == s->failed_num[1]))) {
2343                         /* have disk failed, and we're requested to fetch it;
2344                          * do compute it
2345                          */
2346                         pr_debug("Computing stripe %llu block %d\n",
2347                                (unsigned long long)sh->sector, disk_idx);
2348                         set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2349                         set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2350                         set_bit(R5_Wantcompute, &dev->flags);
2351                         sh->ops.target = disk_idx;
2352                         sh->ops.target2 = -1; /* no 2nd target */
2353                         s->req_compute = 1;
2354                         /* Careful: from this point on 'uptodate' is in the eye
2355                          * of raid_run_ops which services 'compute' operations
2356                          * before writes. R5_Wantcompute flags a block that will
2357                          * be R5_UPTODATE by the time it is needed for a
2358                          * subsequent operation.
2359                          */
2360                         s->uptodate++;
2361                         return 1;
2362                 } else if (s->uptodate == disks-2 && s->failed >= 2) {
2363                         /* Computing 2-failure is *very* expensive; only
2364                          * do it if failed >= 2
2365                          */
2366                         int other;
2367                         for (other = disks; other--; ) {
2368                                 if (other == disk_idx)
2369                                         continue;
2370                                 if (!test_bit(R5_UPTODATE,
2371                                       &sh->dev[other].flags))
2372                                         break;
2373                         }
2374                         BUG_ON(other < 0);
2375                         pr_debug("Computing stripe %llu blocks %d,%d\n",
2376                                (unsigned long long)sh->sector,
2377                                disk_idx, other);
2378                         set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2379                         set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2380                         set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
2381                         set_bit(R5_Wantcompute, &sh->dev[other].flags);
2382                         sh->ops.target = disk_idx;
2383                         sh->ops.target2 = other;
2384                         s->uptodate += 2;
2385                         s->req_compute = 1;
2386                         return 1;
2387                 } else if (test_bit(R5_Insync, &dev->flags)) {
2388                         set_bit(R5_LOCKED, &dev->flags);
2389                         set_bit(R5_Wantread, &dev->flags);
2390                         s->locked++;
2391                         pr_debug("Reading block %d (sync=%d)\n",
2392                                 disk_idx, s->syncing);
2393                 }
2394         }
2395
2396         return 0;
2397 }
2398
2399 /**
2400  * handle_stripe_fill - read or compute data to satisfy pending requests.
2401  */
2402 static void handle_stripe_fill(struct stripe_head *sh,
2403                                struct stripe_head_state *s,
2404                                int disks)
2405 {
2406         int i;
2407
2408         /* look for blocks to read/compute, skip this if a compute
2409          * is already in flight, or if the stripe contents are in the
2410          * midst of changing due to a write
2411          */
2412         if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
2413             !sh->reconstruct_state)
2414                 for (i = disks; i--; )
2415                         if (fetch_block(sh, s, i, disks))
2416                                 break;
2417         set_bit(STRIPE_HANDLE, &sh->state);
2418 }
2419
2420
2421 /* handle_stripe_clean_event
2422  * any written block on an uptodate or failed drive can be returned.
2423  * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
2424  * never LOCKED, so we don't need to test 'failed' directly.
2425  */
2426 static void handle_stripe_clean_event(raid5_conf_t *conf,
2427         struct stripe_head *sh, int disks, struct bio **return_bi)
2428 {
2429         int i;
2430         struct r5dev *dev;
2431
2432         for (i = disks; i--; )
2433                 if (sh->dev[i].written) {
2434                         dev = &sh->dev[i];
2435                         if (!test_bit(R5_LOCKED, &dev->flags) &&
2436                                 test_bit(R5_UPTODATE, &dev->flags)) {
2437                                 /* We can return any write requests */
2438                                 struct bio *wbi, *wbi2;
2439                                 int bitmap_end = 0;
2440                                 pr_debug("Return write for disc %d\n", i);
2441                                 spin_lock_irq(&conf->device_lock);
2442                                 wbi = dev->written;
2443                                 dev->written = NULL;
2444                                 while (wbi && wbi->bi_sector <
2445                                         dev->sector + STRIPE_SECTORS) {
2446                                         wbi2 = r5_next_bio(wbi, dev->sector);
2447                                         if (!raid5_dec_bi_phys_segments(wbi)) {
2448                                                 md_write_end(conf->mddev);
2449                                                 wbi->bi_next = *return_bi;
2450                                                 *return_bi = wbi;
2451                                         }
2452                                         wbi = wbi2;
2453                                 }
2454                                 if (dev->towrite == NULL)
2455                                         bitmap_end = 1;
2456                                 spin_unlock_irq(&conf->device_lock);
2457                                 if (bitmap_end)
2458                                         bitmap_endwrite(conf->mddev->bitmap,
2459                                                         sh->sector,
2460                                                         STRIPE_SECTORS,
2461                                          !test_bit(STRIPE_DEGRADED, &sh->state),
2462                                                         0);
2463                         }
2464                 }
2465
2466         if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2467                 if (atomic_dec_and_test(&conf->pending_full_writes))
2468                         md_wakeup_thread(conf->mddev->thread);
2469 }
2470
2471 static void handle_stripe_dirtying(raid5_conf_t *conf,
2472                                    struct stripe_head *sh,
2473                                    struct stripe_head_state *s,
2474                                    int disks)
2475 {
2476         int rmw = 0, rcw = 0, i;
2477         if (conf->max_degraded == 2) {
2478                 /* RAID6 requires 'rcw' in current implementation
2479                  * Calculate the real rcw later - for now fake it
2480                  * look like rcw is cheaper
2481                  */
2482                 rcw = 1; rmw = 2;
2483         } else for (i = disks; i--; ) {
2484                 /* would I have to read this buffer for read_modify_write */
2485                 struct r5dev *dev = &sh->dev[i];
2486                 if ((dev->towrite || i == sh->pd_idx) &&
2487                     !test_bit(R5_LOCKED, &dev->flags) &&
2488                     !(test_bit(R5_UPTODATE, &dev->flags) ||
2489                       test_bit(R5_Wantcompute, &dev->flags))) {
2490                         if (test_bit(R5_Insync, &dev->flags))
2491                                 rmw++;
2492                         else
2493                                 rmw += 2*disks;  /* cannot read it */
2494                 }
2495                 /* Would I have to read this buffer for reconstruct_write */
2496                 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
2497                     !test_bit(R5_LOCKED, &dev->flags) &&
2498                     !(test_bit(R5_UPTODATE, &dev->flags) ||
2499                     test_bit(R5_Wantcompute, &dev->flags))) {
2500                         if (test_bit(R5_Insync, &dev->flags)) rcw++;
2501                         else
2502                                 rcw += 2*disks;
2503                 }
2504         }
2505         pr_debug("for sector %llu, rmw=%d rcw=%d\n",
2506                 (unsigned long long)sh->sector, rmw, rcw);
2507         set_bit(STRIPE_HANDLE, &sh->state);
2508         if (rmw < rcw && rmw > 0)
2509                 /* prefer read-modify-write, but need to get some data */
2510                 for (i = disks; i--; ) {
2511                         struct r5dev *dev = &sh->dev[i];
2512                         if ((dev->towrite || i == sh->pd_idx) &&
2513                             !test_bit(R5_LOCKED, &dev->flags) &&
2514                             !(test_bit(R5_UPTODATE, &dev->flags) ||
2515                             test_bit(R5_Wantcompute, &dev->flags)) &&
2516                             test_bit(R5_Insync, &dev->flags)) {
2517                                 if (
2518                                   test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2519                                         pr_debug("Read_old block "
2520                                                 "%d for r-m-w\n", i);
2521                                         set_bit(R5_LOCKED, &dev->flags);
2522                                         set_bit(R5_Wantread, &dev->flags);
2523                                         s->locked++;
2524                                 } else {
2525                                         set_bit(STRIPE_DELAYED, &sh->state);
2526                                         set_bit(STRIPE_HANDLE, &sh->state);
2527                                 }
2528                         }
2529                 }
2530         if (rcw <= rmw && rcw > 0) {
2531                 /* want reconstruct write, but need to get some data */
2532                 rcw = 0;
2533                 for (i = disks; i--; ) {
2534                         struct r5dev *dev = &sh->dev[i];
2535                         if (!test_bit(R5_OVERWRITE, &dev->flags) &&
2536                             i != sh->pd_idx && i != sh->qd_idx &&
2537                             !test_bit(R5_LOCKED, &dev->flags) &&
2538                             !(test_bit(R5_UPTODATE, &dev->flags) ||
2539                               test_bit(R5_Wantcompute, &dev->flags))) {
2540                                 rcw++;
2541                                 if (!test_bit(R5_Insync, &dev->flags))
2542                                         continue; /* it's a failed drive */
2543                                 if (
2544                                   test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2545                                         pr_debug("Read_old block "
2546                                                 "%d for Reconstruct\n", i);
2547                                         set_bit(R5_LOCKED, &dev->flags);
2548                                         set_bit(R5_Wantread, &dev->flags);
2549                                         s->locked++;
2550                                 } else {
2551                                         set_bit(STRIPE_DELAYED, &sh->state);
2552                                         set_bit(STRIPE_HANDLE, &sh->state);
2553                                 }
2554                         }
2555                 }
2556         }
2557         /* now if nothing is locked, and if we have enough data,
2558          * we can start a write request
2559          */
2560         /* since handle_stripe can be called at any time we need to handle the
2561          * case where a compute block operation has been submitted and then a
2562          * subsequent call wants to start a write request.  raid_run_ops only
2563          * handles the case where compute block and reconstruct are requested
2564          * simultaneously.  If this is not the case then new writes need to be
2565          * held off until the compute completes.
2566          */
2567         if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
2568             (s->locked == 0 && (rcw == 0 || rmw == 0) &&
2569             !test_bit(STRIPE_BIT_DELAY, &sh->state)))
2570                 schedule_reconstruction(sh, s, rcw == 0, 0);
2571 }
2572
2573 static void handle_parity_checks5(raid5_conf_t *conf, struct stripe_head *sh,
2574                                 struct stripe_head_state *s, int disks)
2575 {
2576         struct r5dev *dev = NULL;
2577
2578         set_bit(STRIPE_HANDLE, &sh->state);
2579
2580         switch (sh->check_state) {
2581         case check_state_idle:
2582                 /* start a new check operation if there are no failures */
2583                 if (s->failed == 0) {
2584                         BUG_ON(s->uptodate != disks);
2585                         sh->check_state = check_state_run;
2586                         set_bit(STRIPE_OP_CHECK, &s->ops_request);
2587                         clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
2588                         s->uptodate--;
2589                         break;
2590                 }
2591                 dev = &sh->dev[s->failed_num[0]];
2592                 /* fall through */
2593         case check_state_compute_result:
2594                 sh->check_state = check_state_idle;
2595                 if (!dev)
2596                         dev = &sh->dev[sh->pd_idx];
2597
2598                 /* check that a write has not made the stripe insync */
2599                 if (test_bit(STRIPE_INSYNC, &sh->state))
2600                         break;
2601
2602                 /* either failed parity check, or recovery is happening */
2603                 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
2604                 BUG_ON(s->uptodate != disks);
2605
2606                 set_bit(R5_LOCKED, &dev->flags);
2607                 s->locked++;
2608                 set_bit(R5_Wantwrite, &dev->flags);
2609
2610                 clear_bit(STRIPE_DEGRADED, &sh->state);
2611                 set_bit(STRIPE_INSYNC, &sh->state);
2612                 break;
2613         case check_state_run:
2614                 break; /* we will be called again upon completion */
2615         case check_state_check_result:
2616                 sh->check_state = check_state_idle;
2617
2618                 /* if a failure occurred during the check operation, leave
2619                  * STRIPE_INSYNC not set and let the stripe be handled again
2620                  */
2621                 if (s->failed)
2622                         break;
2623
2624                 /* handle a successful check operation, if parity is correct
2625                  * we are done.  Otherwise update the mismatch count and repair
2626                  * parity if !MD_RECOVERY_CHECK
2627                  */
2628                 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
2629                         /* parity is correct (on disc,
2630                          * not in buffer any more)
2631                          */
2632                         set_bit(STRIPE_INSYNC, &sh->state);
2633                 else {
2634                         conf->mddev->resync_mismatches += STRIPE_SECTORS;
2635                         if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
2636                                 /* don't try to repair!! */
2637                                 set_bit(STRIPE_INSYNC, &sh->state);
2638                         else {
2639                                 sh->check_state = check_state_compute_run;
2640                                 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2641                                 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2642                                 set_bit(R5_Wantcompute,
2643                                         &sh->dev[sh->pd_idx].flags);
2644                                 sh->ops.target = sh->pd_idx;
2645                                 sh->ops.target2 = -1;
2646                                 s->uptodate++;
2647                         }
2648                 }
2649                 break;
2650         case check_state_compute_run:
2651                 break;
2652         default:
2653                 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
2654                        __func__, sh->check_state,
2655                        (unsigned long long) sh->sector);
2656                 BUG();
2657         }
2658 }
2659
2660
2661 static void handle_parity_checks6(raid5_conf_t *conf, struct stripe_head *sh,
2662                                   struct stripe_head_state *s,
2663                                   int disks)
2664 {
2665         int pd_idx = sh->pd_idx;
2666         int qd_idx = sh->qd_idx;
2667         struct r5dev *dev;
2668
2669         set_bit(STRIPE_HANDLE, &sh->state);
2670
2671         BUG_ON(s->failed > 2);
2672
2673         /* Want to check and possibly repair P and Q.
2674          * However there could be one 'failed' device, in which
2675          * case we can only check one of them, possibly using the
2676          * other to generate missing data
2677          */
2678
2679         switch (sh->check_state) {
2680         case check_state_idle:
2681                 /* start a new check operation if there are < 2 failures */
2682                 if (s->failed == s->q_failed) {
2683                         /* The only possible failed device holds Q, so it
2684                          * makes sense to check P (If anything else were failed,
2685                          * we would have used P to recreate it).
2686                          */
2687                         sh->check_state = check_state_run;
2688                 }
2689                 if (!s->q_failed && s->failed < 2) {
2690                         /* Q is not failed, and we didn't use it to generate
2691                          * anything, so it makes sense to check it
2692                          */
2693                         if (sh->check_state == check_state_run)
2694                                 sh->check_state = check_state_run_pq;
2695                         else
2696                                 sh->check_state = check_state_run_q;
2697                 }
2698
2699                 /* discard potentially stale zero_sum_result */
2700                 sh->ops.zero_sum_result = 0;
2701
2702                 if (sh->check_state == check_state_run) {
2703                         /* async_xor_zero_sum destroys the contents of P */
2704                         clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2705                         s->uptodate--;
2706                 }
2707                 if (sh->check_state >= check_state_run &&
2708                     sh->check_state <= check_state_run_pq) {
2709                         /* async_syndrome_zero_sum preserves P and Q, so
2710                          * no need to mark them !uptodate here
2711                          */
2712                         set_bit(STRIPE_OP_CHECK, &s->ops_request);
2713                         break;
2714                 }
2715
2716                 /* we have 2-disk failure */
2717                 BUG_ON(s->failed != 2);
2718                 /* fall through */
2719         case check_state_compute_result:
2720                 sh->check_state = check_state_idle;
2721
2722                 /* check that a write has not made the stripe insync */
2723                 if (test_bit(STRIPE_INSYNC, &sh->state))
2724                         break;
2725
2726                 /* now write out any block on a failed drive,
2727                  * or P or Q if they were recomputed
2728                  */
2729                 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
2730                 if (s->failed == 2) {
2731                         dev = &sh->dev[s->failed_num[1]];
2732                         s->locked++;
2733                         set_bit(R5_LOCKED, &dev->flags);
2734                         set_bit(R5_Wantwrite, &dev->flags);
2735                 }
2736                 if (s->failed >= 1) {
2737                         dev = &sh->dev[s->failed_num[0]];
2738                         s->locked++;
2739                         set_bit(R5_LOCKED, &dev->flags);
2740                         set_bit(R5_Wantwrite, &dev->flags);
2741                 }
2742                 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
2743                         dev = &sh->dev[pd_idx];
2744                         s->locked++;
2745                         set_bit(R5_LOCKED, &dev->flags);
2746                         set_bit(R5_Wantwrite, &dev->flags);
2747                 }
2748                 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
2749                         dev = &sh->dev[qd_idx];
2750                         s->locked++;
2751                         set_bit(R5_LOCKED, &dev->flags);
2752                         set_bit(R5_Wantwrite, &dev->flags);
2753                 }
2754                 clear_bit(STRIPE_DEGRADED, &sh->state);
2755
2756                 set_bit(STRIPE_INSYNC, &sh->state);
2757                 break;
2758         case check_state_run:
2759         case check_state_run_q:
2760         case check_state_run_pq:
2761                 break; /* we will be called again upon completion */
2762         case check_state_check_result:
2763                 sh->check_state = check_state_idle;
2764
2765                 /* handle a successful check operation, if parity is correct
2766                  * we are done.  Otherwise update the mismatch count and repair
2767                  * parity if !MD_RECOVERY_CHECK
2768                  */
2769                 if (sh->ops.zero_sum_result == 0) {
2770                         /* both parities are correct */
2771                         if (!s->failed)
2772                                 set_bit(STRIPE_INSYNC, &sh->state);
2773                         else {
2774                                 /* in contrast to the raid5 case we can validate
2775                                  * parity, but still have a failure to write
2776                                  * back
2777                                  */
2778                                 sh->check_state = check_state_compute_result;
2779                                 /* Returning at this point means that we may go
2780                                  * off and bring p and/or q uptodate again so
2781                                  * we make sure to check zero_sum_result again
2782                                  * to verify if p or q need writeback
2783                                  */
2784                         }
2785                 } else {
2786                         conf->mddev->resync_mismatches += STRIPE_SECTORS;
2787                         if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
2788                                 /* don't try to repair!! */
2789                                 set_bit(STRIPE_INSYNC, &sh->state);
2790                         else {
2791                                 int *target = &sh->ops.target;
2792
2793                                 sh->ops.target = -1;
2794                                 sh->ops.target2 = -1;
2795                                 sh->check_state = check_state_compute_run;
2796                                 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2797                                 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2798                                 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
2799                                         set_bit(R5_Wantcompute,
2800                                                 &sh->dev[pd_idx].flags);
2801                                         *target = pd_idx;
2802                                         target = &sh->ops.target2;
2803                                         s->uptodate++;
2804                                 }
2805                                 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
2806                                         set_bit(R5_Wantcompute,
2807                                                 &sh->dev[qd_idx].flags);
2808                                         *target = qd_idx;
2809                                         s->uptodate++;
2810                                 }
2811                         }
2812                 }
2813                 break;
2814         case check_state_compute_run:
2815                 break;
2816         default:
2817                 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
2818                        __func__, sh->check_state,
2819                        (unsigned long long) sh->sector);
2820                 BUG();
2821         }
2822 }
2823
2824 static void handle_stripe_expansion(raid5_conf_t *conf, struct stripe_head *sh)
2825 {
2826         int i;
2827
2828         /* We have read all the blocks in this stripe and now we need to
2829          * copy some of them into a target stripe for expand.
2830          */
2831         struct dma_async_tx_descriptor *tx = NULL;
2832         clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
2833         for (i = 0; i < sh->disks; i++)
2834                 if (i != sh->pd_idx && i != sh->qd_idx) {
2835                         int dd_idx, j;
2836                         struct stripe_head *sh2;
2837                         struct async_submit_ctl submit;
2838
2839                         sector_t bn = compute_blocknr(sh, i, 1);
2840                         sector_t s = raid5_compute_sector(conf, bn, 0,
2841                                                           &dd_idx, NULL);
2842                         sh2 = get_active_stripe(conf, s, 0, 1, 1);
2843                         if (sh2 == NULL)
2844                                 /* so far only the early blocks of this stripe
2845                                  * have been requested.  When later blocks
2846                                  * get requested, we will try again
2847                                  */
2848                                 continue;
2849                         if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
2850                            test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
2851                                 /* must have already done this block */
2852                                 release_stripe(sh2);
2853                                 continue;
2854                         }
2855
2856                         /* place all the copies on one channel */
2857                         init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
2858                         tx = async_memcpy(sh2->dev[dd_idx].page,
2859                                           sh->dev[i].page, 0, 0, STRIPE_SIZE,
2860                                           &submit);
2861
2862                         set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
2863                         set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
2864                         for (j = 0; j < conf->raid_disks; j++)
2865                                 if (j != sh2->pd_idx &&
2866                                     j != sh2->qd_idx &&
2867                                     !test_bit(R5_Expanded, &sh2->dev[j].flags))
2868                                         break;
2869                         if (j == conf->raid_disks) {
2870                                 set_bit(STRIPE_EXPAND_READY, &sh2->state);
2871                                 set_bit(STRIPE_HANDLE, &sh2->state);
2872                         }
2873                         release_stripe(sh2);
2874
2875                 }
2876         /* done submitting copies, wait for them to complete */
2877         if (tx) {
2878                 async_tx_ack(tx);
2879                 dma_wait_for_async_tx(tx);
2880         }
2881 }
2882
2883
2884 /*
2885  * handle_stripe - do things to a stripe.
2886  *
2887  * We lock the stripe and then examine the state of various bits
2888  * to see what needs to be done.
2889  * Possible results:
2890  *    return some read request which now have data
2891  *    return some write requests which are safely on disc
2892  *    schedule a read on some buffers
2893  *    schedule a write of some buffers
2894  *    return confirmation of parity correctness
2895  *
2896  * buffers are taken off read_list or write_list, and bh_cache buffers
2897  * get BH_Lock set before the stripe lock is released.
2898  *
2899  */
2900
2901 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
2902 {
2903         raid5_conf_t *conf = sh->raid_conf;
2904         int disks = sh->disks;
2905         struct r5dev *dev;
2906         int i;
2907
2908         memset(s, 0, sizeof(*s));
2909
2910         s->syncing = test_bit(STRIPE_SYNCING, &sh->state);
2911         s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
2912         s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
2913         s->failed_num[0] = -1;
2914         s->failed_num[1] = -1;
2915
2916         /* Now to look around and see what can be done */
2917         rcu_read_lock();
2918         spin_lock_irq(&conf->device_lock);
2919         for (i=disks; i--; ) {
2920                 mdk_rdev_t *rdev;
2921
2922                 dev = &sh->dev[i];
2923
2924                 pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
2925                         i, dev->flags, dev->toread, dev->towrite, dev->written);
2926                 /* maybe we can reply to a read
2927                  *
2928                  * new wantfill requests are only permitted while
2929                  * ops_complete_biofill is guaranteed to be inactive
2930                  */
2931                 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
2932                     !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
2933                         set_bit(R5_Wantfill, &dev->flags);
2934
2935                 /* now count some things */
2936                 if (test_bit(R5_LOCKED, &dev->flags))
2937                         s->locked++;
2938                 if (test_bit(R5_UPTODATE, &dev->flags))
2939                         s->uptodate++;
2940                 if (test_bit(R5_Wantcompute, &dev->flags)) {
2941                         s->compute++;
2942                         BUG_ON(s->compute > 2);
2943                 }
2944
2945                 if (test_bit(R5_Wantfill, &dev->flags))
2946                         s->to_fill++;
2947                 else if (dev->toread)
2948                         s->to_read++;
2949                 if (dev->towrite) {
2950                         s->to_write++;
2951                         if (!test_bit(R5_OVERWRITE, &dev->flags))
2952                                 s->non_overwrite++;
2953                 }
2954                 if (dev->written)
2955                         s->written++;
2956                 rdev = rcu_dereference(conf->disks[i].rdev);
2957                 if (s->blocked_rdev == NULL &&
2958                     rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
2959                         s->blocked_rdev = rdev;
2960                         atomic_inc(&rdev->nr_pending);
2961                 }
2962                 clear_bit(R5_Insync, &dev->flags);
2963                 if (!rdev)
2964                         /* Not in-sync */;
2965                 else if (test_bit(In_sync, &rdev->flags))
2966                         set_bit(R5_Insync, &dev->flags);
2967                 else {
2968                         /* in sync if before recovery_offset */
2969                         if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
2970                                 set_bit(R5_Insync, &dev->flags);
2971                 }
2972                 if (!test_bit(R5_Insync, &dev->flags)) {
2973                         /* The ReadError flag will just be confusing now */
2974                         clear_bit(R5_ReadError, &dev->flags);
2975                         clear_bit(R5_ReWrite, &dev->flags);
2976                 }
2977                 if (test_bit(R5_ReadError, &dev->flags))
2978                         clear_bit(R5_Insync, &dev->flags);
2979                 if (!test_bit(R5_Insync, &dev->flags)) {
2980                         if (s->failed < 2)
2981                                 s->failed_num[s->failed] = i;
2982                         s->failed++;
2983                 }
2984         }
2985         spin_unlock_irq(&conf->device_lock);
2986         rcu_read_unlock();
2987 }
2988
2989 static void handle_stripe(struct stripe_head *sh)
2990 {
2991         struct stripe_head_state s;
2992         raid5_conf_t *conf = sh->raid_conf;
2993         int i;
2994         int prexor;
2995         int disks = sh->disks;
2996         struct r5dev *pdev, *qdev;
2997
2998         clear_bit(STRIPE_HANDLE, &sh->state);
2999         if (test_and_set_bit(STRIPE_ACTIVE, &sh->state)) {
3000                 /* already being handled, ensure it gets handled
3001                  * again when current action finishes */
3002                 set_bit(STRIPE_HANDLE, &sh->state);
3003                 return;
3004         }
3005
3006         if (test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
3007                 set_bit(STRIPE_SYNCING, &sh->state);
3008                 clear_bit(STRIPE_INSYNC, &sh->state);
3009         }
3010         clear_bit(STRIPE_DELAYED, &sh->state);
3011
3012         pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
3013                 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
3014                (unsigned long long)sh->sector, sh->state,
3015                atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
3016                sh->check_state, sh->reconstruct_state);
3017
3018         analyse_stripe(sh, &s);
3019
3020         if (unlikely(s.blocked_rdev)) {
3021                 if (s.syncing || s.expanding || s.expanded ||
3022                     s.to_write || s.written) {
3023                         set_bit(STRIPE_HANDLE, &sh->state);
3024                         goto finish;
3025                 }
3026                 /* There is nothing for the blocked_rdev to block */
3027                 rdev_dec_pending(s.blocked_rdev, conf->mddev);
3028                 s.blocked_rdev = NULL;
3029         }
3030
3031         if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
3032                 set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
3033                 set_bit(STRIPE_BIOFILL_RUN, &sh->state);
3034         }
3035
3036         pr_debug("locked=%d uptodate=%d to_read=%d"
3037                " to_write=%d failed=%d failed_num=%d,%d\n",
3038                s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
3039                s.failed_num[0], s.failed_num[1]);
3040         /* check if the array has lost more than max_degraded devices and,
3041          * if so, some requests might need to be failed.
3042          */
3043         if (s.failed > conf->max_degraded && s.to_read+s.to_write+s.written)
3044                 handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
3045         if (s.failed > conf->max_degraded && s.syncing) {
3046                 md_done_sync(conf->mddev, STRIPE_SECTORS, 0);
3047                 clear_bit(STRIPE_SYNCING, &sh->state);
3048                 s.syncing = 0;
3049         }
3050
3051         /*
3052          * might be able to return some write requests if the parity blocks
3053          * are safe, or on a failed drive
3054          */
3055         pdev = &sh->dev[sh->pd_idx];
3056         s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
3057                 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
3058         qdev = &sh->dev[sh->qd_idx];
3059         s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
3060                 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
3061                 || conf->level < 6;
3062
3063         if (s.written &&
3064             (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
3065                              && !test_bit(R5_LOCKED, &pdev->flags)
3066                              && test_bit(R5_UPTODATE, &pdev->flags)))) &&
3067             (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
3068                              && !test_bit(R5_LOCKED, &qdev->flags)
3069                              && test_bit(R5_UPTODATE, &qdev->flags)))))
3070                 handle_stripe_clean_event(conf, sh, disks, &s.return_bi);
3071
3072         /* Now we might consider reading some blocks, either to check/generate
3073          * parity, or to satisfy requests
3074          * or to load a block that is being partially written.
3075          */
3076         if (s.to_read || s.non_overwrite
3077             || (conf->level == 6 && s.to_write && s.failed)
3078             || (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding)
3079                 handle_stripe_fill(sh, &s, disks);
3080
3081         /* Now we check to see if any write operations have recently
3082          * completed
3083          */
3084         prexor = 0;
3085         if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
3086                 prexor = 1;
3087         if (sh->reconstruct_state == reconstruct_state_drain_result ||
3088             sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
3089                 sh->reconstruct_state = reconstruct_state_idle;
3090
3091                 /* All the 'written' buffers and the parity block are ready to
3092                  * be written back to disk
3093                  */
3094                 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags));
3095                 BUG_ON(sh->qd_idx >= 0 &&
3096                        !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags));
3097                 for (i = disks; i--; ) {
3098                         struct r5dev *dev = &sh->dev[i];
3099                         if (test_bit(R5_LOCKED, &dev->flags) &&
3100                                 (i == sh->pd_idx || i == sh->qd_idx ||
3101                                  dev->written)) {
3102                                 pr_debug("Writing block %d\n", i);
3103                                 set_bit(R5_Wantwrite, &dev->flags);
3104                                 if (prexor)
3105                                         continue;
3106                                 if (!test_bit(R5_Insync, &dev->flags) ||
3107                                     ((i == sh->pd_idx || i == sh->qd_idx)  &&
3108                                      s.failed == 0))
3109                                         set_bit(STRIPE_INSYNC, &sh->state);
3110                         }
3111                 }
3112                 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3113                         s.dec_preread_active = 1;
3114         }
3115
3116         /* Now to consider new write requests and what else, if anything
3117          * should be read.  We do not handle new writes when:
3118          * 1/ A 'write' operation (copy+xor) is already in flight.
3119          * 2/ A 'check' operation is in flight, as it may clobber the parity
3120          *    block.
3121          */
3122         if (s.to_write && !sh->reconstruct_state && !sh->check_state)
3123                 handle_stripe_dirtying(conf, sh, &s, disks);
3124
3125         /* maybe we need to check and possibly fix the parity for this stripe
3126          * Any reads will already have been scheduled, so we just see if enough
3127          * data is available.  The parity check is held off while parity
3128          * dependent operations are in flight.
3129          */
3130         if (sh->check_state ||
3131             (s.syncing && s.locked == 0 &&
3132              !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
3133              !test_bit(STRIPE_INSYNC, &sh->state))) {
3134                 if (conf->level == 6)
3135                         handle_parity_checks6(conf, sh, &s, disks);
3136                 else
3137                         handle_parity_checks5(conf, sh, &s, disks);
3138         }
3139
3140         if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) {
3141                 md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
3142                 clear_bit(STRIPE_SYNCING, &sh->state);
3143         }
3144
3145         /* If the failed drives are just a ReadError, then we might need
3146          * to progress the repair/check process
3147          */
3148         if (s.failed <= conf->max_degraded && !conf->mddev->ro)
3149                 for (i = 0; i < s.failed; i++) {
3150                         struct r5dev *dev = &sh->dev[s.failed_num[i]];
3151                         if (test_bit(R5_ReadError, &dev->flags)
3152                             && !test_bit(R5_LOCKED, &dev->flags)
3153                             && test_bit(R5_UPTODATE, &dev->flags)
3154                                 ) {
3155                                 if (!test_bit(R5_ReWrite, &dev->flags)) {
3156                                         set_bit(R5_Wantwrite, &dev->flags);
3157                                         set_bit(R5_ReWrite, &dev->flags);
3158                                         set_bit(R5_LOCKED, &dev->flags);
3159                                         s.locked++;
3160                                 } else {
3161                                         /* let's read it back */
3162                                         set_bit(R5_Wantread, &dev->flags);
3163                                         set_bit(R5_LOCKED, &dev->flags);
3164                                         s.locked++;
3165                                 }
3166                         }
3167                 }
3168
3169
3170         /* Finish reconstruct operations initiated by the expansion process */
3171         if (sh->reconstruct_state == reconstruct_state_result) {
3172                 struct stripe_head *sh_src
3173                         = get_active_stripe(conf, sh->sector, 1, 1, 1);
3174                 if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
3175                         /* sh cannot be written until sh_src has been read.
3176                          * so arrange for sh to be delayed a little
3177                          */
3178                         set_bit(STRIPE_DELAYED, &sh->state);
3179                         set_bit(STRIPE_HANDLE, &sh->state);
3180                         if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
3181                                               &sh_src->state))
3182                                 atomic_inc(&conf->preread_active_stripes);
3183                         release_stripe(sh_src);
3184                         goto finish;
3185                 }
3186                 if (sh_src)
3187                         release_stripe(sh_src);
3188
3189                 sh->reconstruct_state = reconstruct_state_idle;
3190                 clear_bit(STRIPE_EXPANDING, &sh->state);
3191                 for (i = conf->raid_disks; i--; ) {
3192                         set_bit(R5_Wantwrite, &sh->dev[i].flags);
3193                         set_bit(R5_LOCKED, &sh->dev[i].flags);
3194                         s.locked++;
3195                 }
3196         }
3197
3198         if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
3199             !sh->reconstruct_state) {
3200                 /* Need to write out all blocks after computing parity */
3201                 sh->disks = conf->raid_disks;
3202                 stripe_set_idx(sh->sector, conf, 0, sh);
3203                 schedule_reconstruction(sh, &s, 1, 1);
3204         } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
3205                 clear_bit(STRIPE_EXPAND_READY, &sh->state);
3206                 atomic_dec(&conf->reshape_stripes);
3207                 wake_up(&conf->wait_for_overlap);
3208                 md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
3209         }
3210
3211         if (s.expanding && s.locked == 0 &&
3212             !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
3213                 handle_stripe_expansion(conf, sh);
3214
3215 finish:
3216         /* wait for this device to become unblocked */
3217         if (unlikely(s.blocked_rdev))
3218                 md_wait_for_blocked_rdev(s.blocked_rdev, conf->mddev);
3219
3220         if (s.ops_request)
3221                 raid_run_ops(sh, s.ops_request);
3222
3223         ops_run_io(sh, &s);
3224
3225
3226         if (s.dec_preread_active) {
3227                 /* We delay this until after ops_run_io so that if make_request
3228                  * is waiting on a flush, it won't continue until the writes
3229                  * have actually been submitted.
3230                  */
3231                 atomic_dec(&conf->preread_active_stripes);
3232                 if (atomic_read(&conf->preread_active_stripes) <
3233                     IO_THRESHOLD)
3234                         md_wakeup_thread(conf->mddev->thread);
3235         }
3236
3237         return_io(s.return_bi);
3238
3239         clear_bit(STRIPE_ACTIVE, &sh->state);
3240 }
3241
3242 static void raid5_activate_delayed(raid5_conf_t *conf)
3243 {
3244         if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
3245                 while (!list_empty(&conf->delayed_list)) {
3246                         struct list_head *l = conf->delayed_list.next;
3247                         struct stripe_head *sh;
3248                         sh = list_entry(l, struct stripe_head, lru);
3249                         list_del_init(l);
3250                         clear_bit(STRIPE_DELAYED, &sh->state);
3251                         if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3252                                 atomic_inc(&conf->preread_active_stripes);
3253                         list_add_tail(&sh->lru, &conf->hold_list);
3254                 }
3255         }
3256 }
3257
3258 static void activate_bit_delay(raid5_conf_t *conf)
3259 {
3260         /* device_lock is held */
3261         struct list_head head;
3262         list_add(&head, &conf->bitmap_list);
3263         list_del_init(&conf->bitmap_list);
3264         while (!list_empty(&head)) {
3265                 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
3266                 list_del_init(&sh->lru);
3267                 atomic_inc(&sh->count);
3268                 __release_stripe(conf, sh);
3269         }
3270 }
3271
3272 int md_raid5_congested(mddev_t *mddev, int bits)
3273 {
3274         raid5_conf_t *conf = mddev->private;
3275
3276         /* No difference between reads and writes.  Just check
3277          * how busy the stripe_cache is
3278          */
3279
3280         if (conf->inactive_blocked)
3281                 return 1;
3282         if (conf->quiesce)
3283                 return 1;
3284         if (list_empty_careful(&conf->inactive_list))
3285                 return 1;
3286
3287         return 0;
3288 }
3289 EXPORT_SYMBOL_GPL(md_raid5_congested);
3290
3291 static int raid5_congested(void *data, int bits)
3292 {
3293         mddev_t *mddev = data;
3294
3295         return mddev_congested(mddev, bits) ||
3296                 md_raid5_congested(mddev, bits);
3297 }
3298
3299 /* We want read requests to align with chunks where possible,
3300  * but write requests don't need to.
3301  */
3302 static int raid5_mergeable_bvec(struct request_queue *q,
3303                                 struct bvec_merge_data *bvm,
3304                                 struct bio_vec *biovec)
3305 {
3306         mddev_t *mddev = q->queuedata;
3307         sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
3308         int max;
3309         unsigned int chunk_sectors = mddev->chunk_sectors;
3310         unsigned int bio_sectors = bvm->bi_size >> 9;
3311
3312         if ((bvm->bi_rw & 1) == WRITE)
3313                 return biovec->bv_len; /* always allow writes to be mergeable */
3314
3315         if (mddev->new_chunk_sectors < mddev->chunk_sectors)
3316                 chunk_sectors = mddev->new_chunk_sectors;
3317         max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
3318         if (max < 0) max = 0;
3319         if (max <= biovec->bv_len && bio_sectors == 0)
3320                 return biovec->bv_len;
3321         else
3322                 return max;
3323 }
3324
3325
3326 static int in_chunk_boundary(mddev_t *mddev, struct bio *bio)
3327 {
3328         sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
3329         unsigned int chunk_sectors = mddev->chunk_sectors;
3330         unsigned int bio_sectors = bio->bi_size >> 9;
3331
3332         if (mddev->new_chunk_sectors < mddev->chunk_sectors)
3333                 chunk_sectors = mddev->new_chunk_sectors;
3334         return  chunk_sectors >=
3335                 ((sector & (chunk_sectors - 1)) + bio_sectors);
3336 }
3337
3338 /*
3339  *  add bio to the retry LIFO  ( in O(1) ... we are in interrupt )
3340  *  later sampled by raid5d.
3341  */
3342 static void add_bio_to_retry(struct bio *bi,raid5_conf_t *conf)
3343 {
3344         unsigned long flags;
3345
3346         spin_lock_irqsave(&conf->device_lock, flags);
3347
3348         bi->bi_next = conf->retry_read_aligned_list;
3349         conf->retry_read_aligned_list = bi;
3350
3351         spin_unlock_irqrestore(&conf->device_lock, flags);
3352         md_wakeup_thread(conf->mddev->thread);
3353 }
3354
3355
3356 static struct bio *remove_bio_from_retry(raid5_conf_t *conf)
3357 {
3358         struct bio *bi;
3359
3360         bi = conf->retry_read_aligned;
3361         if (bi) {
3362                 conf->retry_read_aligned = NULL;
3363                 return bi;
3364         }
3365         bi = conf->retry_read_aligned_list;
3366         if(bi) {
3367                 conf->retry_read_aligned_list = bi->bi_next;
3368                 bi->bi_next = NULL;
3369                 /*
3370                  * this sets the active strip count to 1 and the processed
3371                  * strip count to zero (upper 8 bits)
3372                  */
3373                 bi->bi_phys_segments = 1; /* biased count of active stripes */
3374         }
3375
3376         return bi;
3377 }
3378
3379
3380 /*
3381  *  The "raid5_align_endio" should check if the read succeeded and if it
3382  *  did, call bio_endio on the original bio (having bio_put the new bio
3383  *  first).
3384  *  If the read failed..
3385  */
3386 static void raid5_align_endio(struct bio *bi, int error)
3387 {
3388         struct bio* raid_bi  = bi->bi_private;
3389         mddev_t *mddev;
3390         raid5_conf_t *conf;
3391         int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
3392         mdk_rdev_t *rdev;
3393
3394         bio_put(bi);
3395
3396         rdev = (void*)raid_bi->bi_next;
3397         raid_bi->bi_next = NULL;
3398         mddev = rdev->mddev;
3399         conf = mddev->private;
3400
3401         rdev_dec_pending(rdev, conf->mddev);
3402
3403         if (!error && uptodate) {
3404                 bio_endio(raid_bi, 0);
3405                 if (atomic_dec_and_test(&conf->active_aligned_reads))
3406                         wake_up(&conf->wait_for_stripe);
3407                 return;
3408         }
3409
3410
3411         pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
3412
3413         add_bio_to_retry(raid_bi, conf);
3414 }
3415
3416 static int bio_fits_rdev(struct bio *bi)
3417 {
3418         struct request_queue *q = bdev_get_queue(bi->bi_bdev);
3419
3420         if ((bi->bi_size>>9) > queue_max_sectors(q))
3421                 return 0;
3422         blk_recount_segments(q, bi);
3423         if (bi->bi_phys_segments > queue_max_segments(q))
3424                 return 0;
3425
3426         if (q->merge_bvec_fn)
3427                 /* it's too hard to apply the merge_bvec_fn at this stage,
3428                  * just just give up
3429                  */
3430                 return 0;
3431
3432         return 1;
3433 }
3434
3435
3436 static int chunk_aligned_read(mddev_t *mddev, struct bio * raid_bio)
3437 {
3438         raid5_conf_t *conf = mddev->private;
3439         int dd_idx;
3440         struct bio* align_bi;
3441         mdk_rdev_t *rdev;
3442
3443         if (!in_chunk_boundary(mddev, raid_bio)) {
3444                 pr_debug("chunk_aligned_read : non aligned\n");
3445                 return 0;
3446         }
3447         /*
3448          * use bio_clone_mddev to make a copy of the bio
3449          */
3450         align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
3451         if (!align_bi)
3452                 return 0;
3453         /*
3454          *   set bi_end_io to a new function, and set bi_private to the
3455          *     original bio.
3456          */
3457         align_bi->bi_end_io  = raid5_align_endio;
3458         align_bi->bi_private = raid_bio;
3459         /*
3460          *      compute position
3461          */
3462         align_bi->bi_sector =  raid5_compute_sector(conf, raid_bio->bi_sector,
3463                                                     0,
3464                                                     &dd_idx, NULL);
3465
3466         rcu_read_lock();
3467         rdev = rcu_dereference(conf->disks[dd_idx].rdev);
3468         if (rdev && test_bit(In_sync, &rdev->flags)) {
3469                 atomic_inc(&rdev->nr_pending);
3470                 rcu_read_unlock();
3471                 raid_bio->bi_next = (void*)rdev;
3472                 align_bi->bi_bdev =  rdev->bdev;
3473                 align_bi->bi_flags &= ~(1 << BIO_SEG_VALID);
3474                 align_bi->bi_sector += rdev->data_offset;
3475
3476                 if (!bio_fits_rdev(align_bi)) {
3477                         /* too big in some way */
3478                         bio_put(align_bi);
3479                         rdev_dec_pending(rdev, mddev);
3480                         return 0;
3481                 }
3482
3483                 spin_lock_irq(&conf->device_lock);
3484                 wait_event_lock_irq(conf->wait_for_stripe,
3485                                     conf->quiesce == 0,
3486                                     conf->device_lock, /* nothing */);
3487                 atomic_inc(&conf->active_aligned_reads);
3488                 spin_unlock_irq(&conf->device_lock);
3489
3490                 generic_make_request(align_bi);
3491                 return 1;
3492         } else {
3493                 rcu_read_unlock();
3494                 bio_put(align_bi);
3495                 return 0;
3496         }
3497 }
3498
3499 /* __get_priority_stripe - get the next stripe to process
3500  *
3501  * Full stripe writes are allowed to pass preread active stripes up until
3502  * the bypass_threshold is exceeded.  In general the bypass_count
3503  * increments when the handle_list is handled before the hold_list; however, it
3504  * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
3505  * stripe with in flight i/o.  The bypass_count will be reset when the
3506  * head of the hold_list has changed, i.e. the head was promoted to the
3507  * handle_list.
3508  */
3509 static struct stripe_head *__get_priority_stripe(raid5_conf_t *conf)
3510 {
3511         struct stripe_head *sh;
3512
3513         pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
3514                   __func__,
3515                   list_empty(&conf->handle_list) ? "empty" : "busy",
3516                   list_empty(&conf->hold_list) ? "empty" : "busy",
3517                   atomic_read(&conf->pending_full_writes), conf->bypass_count);
3518
3519         if (!list_empty(&conf->handle_list)) {
3520                 sh = list_entry(conf->handle_list.next, typeof(*sh), lru);
3521
3522                 if (list_empty(&conf->hold_list))
3523                         conf->bypass_count = 0;
3524                 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
3525                         if (conf->hold_list.next == conf->last_hold)
3526                                 conf->bypass_count++;
3527                         else {
3528                                 conf->last_hold = conf->hold_list.next;
3529                                 conf->bypass_count -= conf->bypass_threshold;
3530                                 if (conf->bypass_count < 0)
3531                                         conf->bypass_count = 0;
3532                         }
3533                 }
3534         } else if (!list_empty(&conf->hold_list) &&
3535                    ((conf->bypass_threshold &&
3536                      conf->bypass_count > conf->bypass_threshold) ||
3537                     atomic_read(&conf->pending_full_writes) == 0)) {
3538                 sh = list_entry(conf->hold_list.next,
3539                                 typeof(*sh), lru);
3540                 conf->bypass_count -= conf->bypass_threshold;
3541                 if (conf->bypass_count < 0)
3542                         conf->bypass_count = 0;
3543         } else
3544                 return NULL;
3545
3546         list_del_init(&sh->lru);
3547         atomic_inc(&sh->count);
3548         BUG_ON(atomic_read(&sh->count) != 1);
3549         return sh;
3550 }
3551
3552 static int make_request(mddev_t *mddev, struct bio * bi)
3553 {
3554         raid5_conf_t *conf = mddev->private;
3555         int dd_idx;
3556         sector_t new_sector;
3557         sector_t logical_sector, last_sector;
3558         struct stripe_head *sh;
3559         const int rw = bio_data_dir(bi);
3560         int remaining;
3561         int plugged;
3562
3563         if (unlikely(bi->bi_rw & REQ_FLUSH)) {
3564                 md_flush_request(mddev, bi);
3565                 return 0;
3566         }
3567
3568         md_write_start(mddev, bi);
3569
3570         if (rw == READ &&
3571              mddev->reshape_position == MaxSector &&
3572              chunk_aligned_read(mddev,bi))
3573                 return 0;
3574
3575         logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
3576         last_sector = bi->bi_sector + (bi->bi_size>>9);
3577         bi->bi_next = NULL;
3578         bi->bi_phys_segments = 1;       /* over-loaded to count active stripes */
3579
3580         plugged = mddev_check_plugged(mddev);
3581         for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
3582                 DEFINE_WAIT(w);
3583                 int disks, data_disks;
3584                 int previous;
3585
3586         retry:
3587                 previous = 0;
3588                 disks = conf->raid_disks;
3589                 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
3590                 if (unlikely(conf->reshape_progress != MaxSector)) {
3591                         /* spinlock is needed as reshape_progress may be
3592                          * 64bit on a 32bit platform, and so it might be
3593                          * possible to see a half-updated value
3594                          * Of course reshape_progress could change after
3595                          * the lock is dropped, so once we get a reference
3596                          * to the stripe that we think it is, we will have
3597                          * to check again.
3598                          */
3599                         spin_lock_irq(&conf->device_lock);
3600                         if (mddev->delta_disks < 0
3601                             ? logical_sector < conf->reshape_progress
3602                             : logical_sector >= conf->reshape_progress) {
3603                                 disks = conf->previous_raid_disks;
3604                                 previous = 1;
3605                         } else {
3606                                 if (mddev->delta_disks < 0
3607                                     ? logical_sector < conf->reshape_safe
3608                                     : logical_sector >= conf->reshape_safe) {
3609                                         spin_unlock_irq(&conf->device_lock);
3610                                         schedule();
3611                                         goto retry;
3612                                 }
3613                         }
3614                         spin_unlock_irq(&conf->device_lock);
3615                 }
3616                 data_disks = disks - conf->max_degraded;
3617
3618                 new_sector = raid5_compute_sector(conf, logical_sector,
3619                                                   previous,
3620                                                   &dd_idx, NULL);
3621                 pr_debug("raid456: make_request, sector %llu logical %llu\n",
3622                         (unsigned long long)new_sector, 
3623                         (unsigned long long)logical_sector);
3624
3625                 sh = get_active_stripe(conf, new_sector, previous,
3626                                        (bi->bi_rw&RWA_MASK), 0);
3627                 if (sh) {
3628                         if (unlikely(previous)) {
3629                                 /* expansion might have moved on while waiting for a
3630                                  * stripe, so we must do the range check again.
3631                                  * Expansion could still move past after this
3632                                  * test, but as we are holding a reference to
3633                                  * 'sh', we know that if that happens,
3634                                  *  STRIPE_EXPANDING will get set and the expansion
3635                                  * won't proceed until we finish with the stripe.
3636                                  */
3637                                 int must_retry = 0;
3638                                 spin_lock_irq(&conf->device_lock);
3639                                 if (mddev->delta_disks < 0
3640                                     ? logical_sector >= conf->reshape_progress
3641                                     : logical_sector < conf->reshape_progress)
3642                                         /* mismatch, need to try again */
3643                                         must_retry = 1;
3644                                 spin_unlock_irq(&conf->device_lock);
3645                                 if (must_retry) {
3646                                         release_stripe(sh);
3647                                         schedule();
3648                                         goto retry;
3649                                 }
3650                         }
3651
3652                         if (rw == WRITE &&
3653                             logical_sector >= mddev->suspend_lo &&
3654                             logical_sector < mddev->suspend_hi) {
3655                                 release_stripe(sh);
3656                                 /* As the suspend_* range is controlled by
3657                                  * userspace, we want an interruptible
3658                                  * wait.
3659                                  */
3660                                 flush_signals(current);
3661                                 prepare_to_wait(&conf->wait_for_overlap,
3662                                                 &w, TASK_INTERRUPTIBLE);
3663                                 if (logical_sector >= mddev->suspend_lo &&
3664                                     logical_sector < mddev->suspend_hi)
3665                                         schedule();
3666                                 goto retry;
3667                         }
3668
3669                         if (test_bit(STRIPE_EXPANDING, &sh->state) ||
3670                             !add_stripe_bio(sh, bi, dd_idx, rw)) {
3671                                 /* Stripe is busy expanding or
3672                                  * add failed due to overlap.  Flush everything
3673                                  * and wait a while
3674                                  */
3675                                 md_wakeup_thread(mddev->thread);
3676                                 release_stripe(sh);
3677                                 schedule();
3678                                 goto retry;
3679                         }
3680                         finish_wait(&conf->wait_for_overlap, &w);
3681                         set_bit(STRIPE_HANDLE, &sh->state);
3682                         clear_bit(STRIPE_DELAYED, &sh->state);
3683                         if ((bi->bi_rw & REQ_SYNC) &&
3684                             !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3685                                 atomic_inc(&conf->preread_active_stripes);
3686                         release_stripe(sh);
3687                 } else {
3688                         /* cannot get stripe for read-ahead, just give-up */
3689                         clear_bit(BIO_UPTODATE, &bi->bi_flags);
3690                         finish_wait(&conf->wait_for_overlap, &w);
3691                         break;
3692                 }
3693                         
3694         }
3695         if (!plugged)
3696                 md_wakeup_thread(mddev->thread);
3697
3698         spin_lock_irq(&conf->device_lock);
3699         remaining = raid5_dec_bi_phys_segments(bi);
3700         spin_unlock_irq(&conf->device_lock);
3701         if (remaining == 0) {
3702
3703                 if ( rw == WRITE )
3704                         md_write_end(mddev);
3705
3706                 bio_endio(bi, 0);
3707         }
3708
3709         return 0;
3710 }
3711
3712 static sector_t raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks);
3713
3714 static sector_t reshape_request(mddev_t *mddev, sector_t sector_nr, int *skipped)
3715 {
3716         /* reshaping is quite different to recovery/resync so it is
3717          * handled quite separately ... here.
3718          *
3719          * On each call to sync_request, we gather one chunk worth of
3720          * destination stripes and flag them as expanding.
3721          * Then we find all the source stripes and request reads.
3722          * As the reads complete, handle_stripe will copy the data
3723          * into the destination stripe and release that stripe.
3724          */
3725         raid5_conf_t *conf = mddev->private;
3726         struct stripe_head *sh;
3727         sector_t first_sector, last_sector;
3728         int raid_disks = conf->previous_raid_disks;
3729         int data_disks = raid_disks - conf->max_degraded;
3730         int new_data_disks = conf->raid_disks - conf->max_degraded;
3731         int i;
3732         int dd_idx;
3733         sector_t writepos, readpos, safepos;
3734         sector_t stripe_addr;
3735         int reshape_sectors;
3736         struct list_head stripes;
3737
3738         if (sector_nr == 0) {
3739                 /* If restarting in the middle, skip the initial sectors */
3740                 if (mddev->delta_disks < 0 &&
3741                     conf->reshape_progress < raid5_size(mddev, 0, 0)) {
3742                         sector_nr = raid5_size(mddev, 0, 0)
3743                                 - conf->reshape_progress;
3744                 } else if (mddev->delta_disks >= 0 &&
3745                            conf->reshape_progress > 0)
3746                         sector_nr = conf->reshape_progress;
3747                 sector_div(sector_nr, new_data_disks);
3748                 if (sector_nr) {
3749                         mddev->curr_resync_completed = sector_nr;
3750                         sysfs_notify(&mddev->kobj, NULL, "sync_completed");
3751                         *skipped = 1;
3752                         return sector_nr;
3753                 }
3754         }
3755
3756         /* We need to process a full chunk at a time.
3757          * If old and new chunk sizes differ, we need to process the
3758          * largest of these
3759          */
3760         if (mddev->new_chunk_sectors > mddev->chunk_sectors)
3761                 reshape_sectors = mddev->new_chunk_sectors;
3762         else
3763                 reshape_sectors = mddev->chunk_sectors;
3764
3765         /* we update the metadata when there is more than 3Meg
3766          * in the block range (that is rather arbitrary, should
3767          * probably be time based) or when the data about to be
3768          * copied would over-write the source of the data at
3769          * the front of the range.
3770          * i.e. one new_stripe along from reshape_progress new_maps
3771          * to after where reshape_safe old_maps to
3772          */
3773         writepos = conf->reshape_progress;
3774         sector_div(writepos, new_data_disks);
3775         readpos = conf->reshape_progress;
3776         sector_div(readpos, data_disks);
3777         safepos = conf->reshape_safe;
3778         sector_div(safepos, data_disks);
3779         if (mddev->delta_disks < 0) {
3780                 writepos -= min_t(sector_t, reshape_sectors, writepos);
3781                 readpos += reshape_sectors;
3782                 safepos += reshape_sectors;
3783         } else {
3784                 writepos += reshape_sectors;
3785                 readpos -= min_t(sector_t, reshape_sectors, readpos);
3786                 safepos -= min_t(sector_t, reshape_sectors, safepos);
3787         }
3788
3789         /* 'writepos' is the most advanced device address we might write.
3790          * 'readpos' is the least advanced device address we might read.
3791          * 'safepos' is the least address recorded in the metadata as having
3792          *     been reshaped.
3793          * If 'readpos' is behind 'writepos', then there is no way that we can
3794          * ensure safety in the face of a crash - that must be done by userspace
3795          * making a backup of the data.  So in that case there is no particular
3796          * rush to update metadata.
3797          * Otherwise if 'safepos' is behind 'writepos', then we really need to
3798          * update the metadata to advance 'safepos' to match 'readpos' so that
3799          * we can be safe in the event of a crash.
3800          * So we insist on updating metadata if safepos is behind writepos and
3801          * readpos is beyond writepos.
3802          * In any case, update the metadata every 10 seconds.
3803          * Maybe that number should be configurable, but I'm not sure it is
3804          * worth it.... maybe it could be a multiple of safemode_delay???
3805          */
3806         if ((mddev->delta_disks < 0
3807              ? (safepos > writepos && readpos < writepos)
3808              : (safepos < writepos && readpos > writepos)) ||
3809             time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
3810                 /* Cannot proceed until we've updated the superblock... */
3811                 wait_event(conf->wait_for_overlap,
3812                            atomic_read(&conf->reshape_stripes)==0);
3813                 mddev->reshape_position = conf->reshape_progress;
3814                 mddev->curr_resync_completed = sector_nr;
3815                 conf->reshape_checkpoint = jiffies;
3816                 set_bit(MD_CHANGE_DEVS, &mddev->flags);
3817                 md_wakeup_thread(mddev->thread);
3818                 wait_event(mddev->sb_wait, mddev->flags == 0 ||
3819                            kthread_should_stop());
3820                 spin_lock_irq(&conf->device_lock);
3821                 conf->reshape_safe = mddev->reshape_position;
3822                 spin_unlock_irq(&conf->device_lock);
3823                 wake_up(&conf->wait_for_overlap);
3824                 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
3825         }
3826
3827         if (mddev->delta_disks < 0) {
3828                 BUG_ON(conf->reshape_progress == 0);
3829                 stripe_addr = writepos;
3830                 BUG_ON((mddev->dev_sectors &
3831                         ~((sector_t)reshape_sectors - 1))
3832                        - reshape_sectors - stripe_addr
3833                        != sector_nr);
3834         } else {
3835                 BUG_ON(writepos != sector_nr + reshape_sectors);
3836                 stripe_addr = sector_nr;
3837         }
3838         INIT_LIST_HEAD(&stripes);
3839         for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
3840                 int j;
3841                 int skipped_disk = 0;
3842                 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
3843                 set_bit(STRIPE_EXPANDING, &sh->state);
3844                 atomic_inc(&conf->reshape_stripes);
3845                 /* If any of this stripe is beyond the end of the old
3846                  * array, then we need to zero those blocks
3847                  */
3848                 for (j=sh->disks; j--;) {
3849                         sector_t s;
3850                         if (j == sh->pd_idx)
3851                                 continue;
3852                         if (conf->level == 6 &&
3853                             j == sh->qd_idx)
3854                                 continue;
3855                         s = compute_blocknr(sh, j, 0);
3856                         if (s < raid5_size(mddev, 0, 0)) {
3857                                 skipped_disk = 1;
3858                                 continue;
3859                         }
3860                         memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
3861                         set_bit(R5_Expanded, &sh->dev[j].flags);
3862                         set_bit(R5_UPTODATE, &sh->dev[j].flags);
3863                 }
3864                 if (!skipped_disk) {
3865                         set_bit(STRIPE_EXPAND_READY, &sh->state);
3866                         set_bit(STRIPE_HANDLE, &sh->state);
3867                 }
3868                 list_add(&sh->lru, &stripes);
3869         }
3870         spin_lock_irq(&conf->device_lock);
3871         if (mddev->delta_disks < 0)
3872                 conf->reshape_progress -= reshape_sectors * new_data_disks;
3873         else
3874                 conf->reshape_progress += reshape_sectors * new_data_disks;
3875         spin_unlock_irq(&conf->device_lock);
3876         /* Ok, those stripe are ready. We can start scheduling
3877          * reads on the source stripes.
3878          * The source stripes are determined by mapping the first and last
3879          * block on the destination stripes.
3880          */
3881         first_sector =
3882                 raid5_compute_sector(conf, stripe_addr*(new_data_disks),
3883                                      1, &dd_idx, NULL);
3884         last_sector =
3885                 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
3886                                             * new_data_disks - 1),
3887                                      1, &dd_idx, NULL);
3888         if (last_sector >= mddev->dev_sectors)
3889                 last_sector = mddev->dev_sectors - 1;
3890         while (first_sector <= last_sector) {
3891                 sh = get_active_stripe(conf, first_sector, 1, 0, 1);
3892                 set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
3893                 set_bit(STRIPE_HANDLE, &sh->state);
3894                 release_stripe(sh);
3895                 first_sector += STRIPE_SECTORS;
3896         }
3897         /* Now that the sources are clearly marked, we can release
3898          * the destination stripes
3899          */
3900         while (!list_empty(&stripes)) {
3901                 sh = list_entry(stripes.next, struct stripe_head, lru);
3902                 list_del_init(&sh->lru);
3903                 release_stripe(sh);
3904         }
3905         /* If this takes us to the resync_max point where we have to pause,
3906          * then we need to write out the superblock.
3907          */
3908         sector_nr += reshape_sectors;
3909         if ((sector_nr - mddev->curr_resync_completed) * 2
3910             >= mddev->resync_max - mddev->curr_resync_completed) {
3911                 /* Cannot proceed until we've updated the superblock... */
3912                 wait_event(conf->wait_for_overlap,
3913                            atomic_read(&conf->reshape_stripes) == 0);
3914                 mddev->reshape_position = conf->reshape_progress;
3915                 mddev->curr_resync_completed = sector_nr;
3916                 conf->reshape_checkpoint = jiffies;
3917                 set_bit(MD_CHANGE_DEVS, &mddev->flags);
3918                 md_wakeup_thread(mddev->thread);
3919                 wait_event(mddev->sb_wait,
3920                            !test_bit(MD_CHANGE_DEVS, &mddev->flags)
3921                            || kthread_should_stop());
3922                 spin_lock_irq(&conf->device_lock);
3923                 conf->reshape_safe = mddev->reshape_position;
3924                 spin_unlock_irq(&conf->device_lock);
3925                 wake_up(&conf->wait_for_overlap);
3926                 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
3927         }
3928         return reshape_sectors;
3929 }
3930
3931 /* FIXME go_faster isn't used */
3932 static inline sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster)
3933 {
3934         raid5_conf_t *conf = mddev->private;
3935         struct stripe_head *sh;
3936         sector_t max_sector = mddev->dev_sectors;
3937         sector_t sync_blocks;
3938         int still_degraded = 0;
3939         int i;
3940
3941         if (sector_nr >= max_sector) {
3942                 /* just being told to finish up .. nothing much to do */
3943
3944                 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
3945                         end_reshape(conf);
3946                         return 0;
3947                 }
3948
3949                 if (mddev->curr_resync < max_sector) /* aborted */
3950                         bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
3951                                         &sync_blocks, 1);
3952                 else /* completed sync */
3953                         conf->fullsync = 0;
3954                 bitmap_close_sync(mddev->bitmap);
3955
3956                 return 0;
3957         }
3958
3959         /* Allow raid5_quiesce to complete */
3960         wait_event(conf->wait_for_overlap, conf->quiesce != 2);
3961
3962         if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
3963                 return reshape_request(mddev, sector_nr, skipped);
3964
3965         /* No need to check resync_max as we never do more than one
3966          * stripe, and as resync_max will always be on a chunk boundary,
3967          * if the check in md_do_sync didn't fire, there is no chance
3968          * of overstepping resync_max here
3969          */
3970
3971         /* if there is too many failed drives and we are trying
3972          * to resync, then assert that we are finished, because there is
3973          * nothing we can do.
3974          */
3975         if (mddev->degraded >= conf->max_degraded &&
3976             test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3977                 sector_t rv = mddev->dev_sectors - sector_nr;
3978                 *skipped = 1;
3979                 return rv;
3980         }
3981         if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
3982             !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
3983             !conf->fullsync && sync_blocks >= STRIPE_SECTORS) {
3984                 /* we can skip this block, and probably more */
3985                 sync_blocks /= STRIPE_SECTORS;
3986                 *skipped = 1;
3987                 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
3988         }
3989
3990
3991         bitmap_cond_end_sync(mddev->bitmap, sector_nr);
3992
3993         sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
3994         if (sh == NULL) {
3995                 sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
3996                 /* make sure we don't swamp the stripe cache if someone else
3997                  * is trying to get access
3998                  */
3999                 schedule_timeout_uninterruptible(1);
4000         }
4001         /* Need to check if array will still be degraded after recovery/resync
4002          * We don't need to check the 'failed' flag as when that gets set,
4003          * recovery aborts.
4004          */
4005         for (i = 0; i < conf->raid_disks; i++)
4006                 if (conf->disks[i].rdev == NULL)
4007                         still_degraded = 1;
4008
4009         bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);
4010
4011         set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
4012
4013         handle_stripe(sh);
4014         release_stripe(sh);
4015
4016         return STRIPE_SECTORS;
4017 }
4018
4019 static int  retry_aligned_read(raid5_conf_t *conf, struct bio *raid_bio)
4020 {
4021         /* We may not be able to submit a whole bio at once as there
4022          * may not be enough stripe_heads available.
4023          * We cannot pre-allocate enough stripe_heads as we may need
4024          * more than exist in the cache (if we allow ever large chunks).
4025          * So we do one stripe head at a time and record in
4026          * ->bi_hw_segments how many have been done.
4027          *
4028          * We *know* that this entire raid_bio is in one chunk, so
4029          * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
4030          */
4031         struct stripe_head *sh;
4032         int dd_idx;
4033         sector_t sector, logical_sector, last_sector;
4034         int scnt = 0;
4035         int remaining;
4036         int handled = 0;
4037
4038         logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
4039         sector = raid5_compute_sector(conf, logical_sector,
4040                                       0, &dd_idx, NULL);
4041         last_sector = raid_bio->bi_sector + (raid_bio->bi_size>>9);
4042
4043         for (; logical_sector < last_sector;
4044              logical_sector += STRIPE_SECTORS,
4045                      sector += STRIPE_SECTORS,
4046                      scnt++) {
4047
4048                 if (scnt < raid5_bi_hw_segments(raid_bio))
4049                         /* already done this stripe */
4050                         continue;
4051
4052                 sh = get_active_stripe(conf, sector, 0, 1, 0);
4053
4054                 if (!sh) {
4055                         /* failed to get a stripe - must wait */
4056                         raid5_set_bi_hw_segments(raid_bio, scnt);
4057                         conf->retry_read_aligned = raid_bio;
4058                         return handled;
4059                 }
4060
4061                 set_bit(R5_ReadError, &sh->dev[dd_idx].flags);
4062                 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) {
4063                         release_stripe(sh);
4064                         raid5_set_bi_hw_segments(raid_bio, scnt);
4065                         conf->retry_read_aligned = raid_bio;
4066                         return handled;
4067                 }
4068
4069                 handle_stripe(sh);
4070                 release_stripe(sh);
4071                 handled++;
4072         }
4073         spin_lock_irq(&conf->device_lock);
4074         remaining = raid5_dec_bi_phys_segments(raid_bio);
4075         spin_unlock_irq(&conf->device_lock);
4076         if (remaining == 0)
4077                 bio_endio(raid_bio, 0);
4078         if (atomic_dec_and_test(&conf->active_aligned_reads))
4079                 wake_up(&conf->wait_for_stripe);
4080         return handled;
4081 }
4082
4083
4084 /*
4085  * This is our raid5 kernel thread.
4086  *
4087  * We scan the hash table for stripes which can be handled now.
4088  * During the scan, completed stripes are saved for us by the interrupt
4089  * handler, so that they will not have to wait for our next wakeup.
4090  */
4091 static void raid5d(mddev_t *mddev)
4092 {
4093         struct stripe_head *sh;
4094         raid5_conf_t *conf = mddev->private;
4095         int handled;
4096         struct blk_plug plug;
4097
4098         pr_debug("+++ raid5d active\n");
4099
4100         md_check_recovery(mddev);
4101
4102         blk_start_plug(&plug);
4103         handled = 0;
4104         spin_lock_irq(&conf->device_lock);
4105         while (1) {
4106                 struct bio *bio;
4107
4108                 if (atomic_read(&mddev->plug_cnt) == 0 &&
4109                     !list_empty(&conf->bitmap_list)) {
4110                         /* Now is a good time to flush some bitmap updates */
4111                         conf->seq_flush++;
4112                         spin_unlock_irq(&conf->device_lock);
4113                         bitmap_unplug(mddev->bitmap);
4114                         spin_lock_irq(&conf->device_lock);
4115                         conf->seq_write = conf->seq_flush;
4116                         activate_bit_delay(conf);
4117                 }
4118                 if (atomic_read(&mddev->plug_cnt) == 0)
4119                         raid5_activate_delayed(conf);
4120
4121                 while ((bio = remove_bio_from_retry(conf))) {
4122                         int ok;
4123                         spin_unlock_irq(&conf->device_lock);
4124                         ok = retry_aligned_read(conf, bio);
4125                         spin_lock_irq(&conf->device_lock);
4126                         if (!ok)
4127                                 break;
4128                         handled++;
4129                 }
4130
4131                 sh = __get_priority_stripe(conf);
4132
4133                 if (!sh)
4134                         break;
4135                 spin_unlock_irq(&conf->device_lock);
4136                 
4137                 handled++;
4138                 handle_stripe(sh);
4139                 release_stripe(sh);
4140                 cond_resched();
4141
4142                 spin_lock_irq(&conf->device_lock);
4143         }
4144         pr_debug("%d stripes handled\n", handled);
4145
4146         spin_unlock_irq(&conf->device_lock);
4147
4148         async_tx_issue_pending_all();
4149         blk_finish_plug(&plug);
4150
4151         pr_debug("--- raid5d inactive\n");
4152 }
4153
4154 static ssize_t
4155 raid5_show_stripe_cache_size(mddev_t *mddev, char *page)
4156 {
4157         raid5_conf_t *conf = mddev->private;
4158         if (conf)
4159                 return sprintf(page, "%d\n", conf->max_nr_stripes);
4160         else
4161                 return 0;
4162 }
4163
4164 int
4165 raid5_set_cache_size(mddev_t *mddev, int size)
4166 {
4167         raid5_conf_t *conf = mddev->private;
4168         int err;
4169
4170         if (size <= 16 || size > 32768)
4171                 return -EINVAL;
4172         while (size < conf->max_nr_stripes) {
4173                 if (drop_one_stripe(conf))
4174                         conf->max_nr_stripes--;
4175                 else
4176                         break;
4177         }
4178         err = md_allow_write(mddev);
4179         if (err)
4180                 return err;
4181         while (size > conf->max_nr_stripes) {
4182                 if (grow_one_stripe(conf))
4183                         conf->max_nr_stripes++;
4184                 else break;
4185         }
4186         return 0;
4187 }
4188 EXPORT_SYMBOL(raid5_set_cache_size);
4189
4190 static ssize_t
4191 raid5_store_stripe_cache_size(mddev_t *mddev, const char *page, size_t len)
4192 {
4193         raid5_conf_t *conf = mddev->private;
4194         unsigned long new;
4195         int err;
4196
4197         if (len >= PAGE_SIZE)
4198                 return -EINVAL;
4199         if (!conf)
4200                 return -ENODEV;
4201
4202         if (strict_strtoul(page, 10, &new))
4203                 return -EINVAL;
4204         err = raid5_set_cache_size(mddev, new);
4205         if (err)
4206                 return err;
4207         return len;
4208 }
4209
4210 static struct md_sysfs_entry
4211 raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
4212                                 raid5_show_stripe_cache_size,
4213                                 raid5_store_stripe_cache_size);
4214
4215 static ssize_t
4216 raid5_show_preread_threshold(mddev_t *mddev, char *page)
4217 {
4218         raid5_conf_t *conf = mddev->private;
4219         if (conf)
4220                 return sprintf(page, "%d\n", conf->bypass_threshold);
4221         else
4222                 return 0;
4223 }
4224
4225 static ssize_t
4226 raid5_store_preread_threshold(mddev_t *mddev, const char *page, size_t len)
4227 {
4228         raid5_conf_t *conf = mddev->private;
4229         unsigned long new;
4230         if (len >= PAGE_SIZE)
4231                 return -EINVAL;
4232         if (!conf)
4233                 return -ENODEV;
4234
4235         if (strict_strtoul(page, 10, &new))
4236                 return -EINVAL;
4237         if (new > conf->max_nr_stripes)
4238                 return -EINVAL;
4239         conf->bypass_threshold = new;
4240         return len;
4241 }
4242
4243 static struct md_sysfs_entry
4244 raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
4245                                         S_IRUGO | S_IWUSR,
4246                                         raid5_show_preread_threshold,
4247                                         raid5_store_preread_threshold);
4248
4249 static ssize_t
4250 stripe_cache_active_show(mddev_t *mddev, char *page)
4251 {
4252         raid5_conf_t *conf = mddev->private;
4253         if (conf)
4254                 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
4255         else
4256                 return 0;
4257 }
4258
4259 static struct md_sysfs_entry
4260 raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
4261
4262 static struct attribute *raid5_attrs[] =  {
4263         &raid5_stripecache_size.attr,
4264         &raid5_stripecache_active.attr,
4265         &raid5_preread_bypass_threshold.attr,
4266         NULL,
4267 };
4268 static struct attribute_group raid5_attrs_group = {
4269         .name = NULL,
4270         .attrs = raid5_attrs,
4271 };
4272
4273 static sector_t
4274 raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks)
4275 {
4276         raid5_conf_t *conf = mddev->private;
4277
4278         if (!sectors)
4279                 sectors = mddev->dev_sectors;
4280         if (!raid_disks)
4281                 /* size is defined by the smallest of previous and new size */
4282                 raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
4283
4284         sectors &= ~((sector_t)mddev->chunk_sectors - 1);
4285         sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
4286         return sectors * (raid_disks - conf->max_degraded);
4287 }
4288
4289 static void raid5_free_percpu(raid5_conf_t *conf)
4290 {
4291         struct raid5_percpu *percpu;
4292         unsigned long cpu;
4293
4294         if (!conf->percpu)
4295                 return;
4296
4297         get_online_cpus();
4298         for_each_possible_cpu(cpu) {
4299                 percpu = per_cpu_ptr(conf->percpu, cpu);
4300                 safe_put_page(percpu->spare_page);
4301                 kfree(percpu->scribble);
4302         }
4303 #ifdef CONFIG_HOTPLUG_CPU
4304         unregister_cpu_notifier(&conf->cpu_notify);
4305 #endif
4306         put_online_cpus();
4307
4308         free_percpu(conf->percpu);
4309 }
4310
4311 static void free_conf(raid5_conf_t *conf)
4312 {
4313         shrink_stripes(conf);
4314         raid5_free_percpu(conf);
4315         kfree(conf->disks);
4316         kfree(conf->stripe_hashtbl);
4317         kfree(conf);
4318 }
4319
4320 #ifdef CONFIG_HOTPLUG_CPU
4321 static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
4322                               void *hcpu)
4323 {
4324         raid5_conf_t *conf = container_of(nfb, raid5_conf_t, cpu_notify);
4325         long cpu = (long)hcpu;
4326         struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
4327
4328         switch (action) {
4329         case CPU_UP_PREPARE:
4330         case CPU_UP_PREPARE_FROZEN:
4331                 if (conf->level == 6 && !percpu->spare_page)
4332                         percpu->spare_page = alloc_page(GFP_KERNEL);
4333                 if (!percpu->scribble)
4334                         percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL);
4335
4336                 if (!percpu->scribble ||
4337                     (conf->level == 6 && !percpu->spare_page)) {
4338                         safe_put_page(percpu->spare_page);
4339                         kfree(percpu->scribble);
4340                         pr_err("%s: failed memory allocation for cpu%ld\n",
4341                                __func__, cpu);
4342                         return notifier_from_errno(-ENOMEM);
4343                 }
4344                 break;
4345         case CPU_DEAD:
4346         case CPU_DEAD_FROZEN:
4347                 safe_put_page(percpu->spare_page);
4348                 kfree(percpu->scribble);
4349                 percpu->spare_page = NULL;
4350                 percpu->scribble = NULL;
4351                 break;
4352         default:
4353                 break;
4354         }
4355         return NOTIFY_OK;
4356 }
4357 #endif
4358
4359 static int raid5_alloc_percpu(raid5_conf_t *conf)
4360 {
4361         unsigned long cpu;
4362         struct page *spare_page;
4363         struct raid5_percpu __percpu *allcpus;
4364         void *scribble;
4365         int err;
4366
4367         allcpus = alloc_percpu(struct raid5_percpu);
4368         if (!allcpus)
4369                 return -ENOMEM;
4370         conf->percpu = allcpus;
4371
4372         get_online_cpus();
4373         err = 0;
4374         for_each_present_cpu(cpu) {
4375                 if (conf->level == 6) {
4376                         spare_page = alloc_page(GFP_KERNEL);
4377                         if (!spare_page) {
4378                                 err = -ENOMEM;
4379                                 break;
4380                         }
4381                         per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page;
4382                 }
4383                 scribble = kmalloc(conf->scribble_len, GFP_KERNEL);
4384                 if (!scribble) {
4385                         err = -ENOMEM;
4386                         break;
4387                 }
4388                 per_cpu_ptr(conf->percpu, cpu)->scribble = scribble;
4389         }
4390 #ifdef CONFIG_HOTPLUG_CPU
4391         conf->cpu_notify.notifier_call = raid456_cpu_notify;
4392         conf->cpu_notify.priority = 0;
4393         if (err == 0)
4394                 err = register_cpu_notifier(&conf->cpu_notify);
4395 #endif
4396         put_online_cpus();
4397
4398         return err;
4399 }
4400
4401 static raid5_conf_t *setup_conf(mddev_t *mddev)
4402 {
4403         raid5_conf_t *conf;
4404         int raid_disk, memory, max_disks;
4405         mdk_rdev_t *rdev;
4406         struct disk_info *disk;
4407
4408         if (mddev->new_level != 5
4409             && mddev->new_level != 4
4410             && mddev->new_level != 6) {
4411                 printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
4412                        mdname(mddev), mddev->new_level);
4413                 return ERR_PTR(-EIO);
4414         }
4415         if ((mddev->new_level == 5
4416              && !algorithm_valid_raid5(mddev->new_layout)) ||
4417             (mddev->new_level == 6
4418              && !algorithm_valid_raid6(mddev->new_layout))) {
4419                 printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
4420                        mdname(mddev), mddev->new_layout);
4421                 return ERR_PTR(-EIO);
4422         }
4423         if (mddev->new_level == 6 && mddev->raid_disks < 4) {
4424                 printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
4425                        mdname(mddev), mddev->raid_disks);
4426                 return ERR_PTR(-EINVAL);
4427         }
4428
4429         if (!mddev->new_chunk_sectors ||
4430             (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
4431             !is_power_of_2(mddev->new_chunk_sectors)) {
4432                 printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
4433                        mdname(mddev), mddev->new_chunk_sectors << 9);
4434                 return ERR_PTR(-EINVAL);
4435         }
4436
4437         conf = kzalloc(sizeof(raid5_conf_t), GFP_KERNEL);
4438         if (conf == NULL)
4439                 goto abort;
4440         spin_lock_init(&conf->device_lock);
4441         init_waitqueue_head(&conf->wait_for_stripe);
4442         init_waitqueue_head(&conf->wait_for_overlap);
4443         INIT_LIST_HEAD(&conf->handle_list);
4444         INIT_LIST_HEAD(&conf->hold_list);
4445         INIT_LIST_HEAD(&conf->delayed_list);
4446         INIT_LIST_HEAD(&conf->bitmap_list);
4447         INIT_LIST_HEAD(&conf->inactive_list);
4448         atomic_set(&conf->active_stripes, 0);
4449         atomic_set(&conf->preread_active_stripes, 0);
4450         atomic_set(&conf->active_aligned_reads, 0);
4451         conf->bypass_threshold = BYPASS_THRESHOLD;
4452
4453         conf->raid_disks = mddev->raid_disks;
4454         if (mddev->reshape_position == MaxSector)
4455                 conf->previous_raid_disks = mddev->raid_disks;
4456         else
4457                 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
4458         max_disks = max(conf->raid_disks, conf->previous_raid_disks);
4459         conf->scribble_len = scribble_len(max_disks);
4460
4461         conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
4462                               GFP_KERNEL);
4463         if (!conf->disks)
4464                 goto abort;
4465
4466         conf->mddev = mddev;
4467
4468         if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
4469                 goto abort;
4470
4471         conf->level = mddev->new_level;
4472         if (raid5_alloc_percpu(conf) != 0)
4473                 goto abort;
4474
4475         pr_debug("raid456: run(%s) called.\n", mdname(mddev));
4476
4477         list_for_each_entry(rdev, &mddev->disks, same_set) {
4478                 raid_disk = rdev->raid_disk;
4479                 if (raid_disk >= max_disks
4480                     || raid_disk < 0)
4481                         continue;
4482                 disk = conf->disks + raid_disk;
4483
4484                 disk->rdev = rdev;
4485
4486                 if (test_bit(In_sync, &rdev->flags)) {
4487                         char b[BDEVNAME_SIZE];
4488                         printk(KERN_INFO "md/raid:%s: device %s operational as raid"
4489                                " disk %d\n",
4490                                mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
4491                 } else if (rdev->saved_raid_disk != raid_disk)
4492                         /* Cannot rely on bitmap to complete recovery */
4493                         conf->fullsync = 1;
4494         }
4495
4496         conf->chunk_sectors = mddev->new_chunk_sectors;
4497         conf->level = mddev->new_level;
4498         if (conf->level == 6)
4499                 conf->max_degraded = 2;
4500         else
4501                 conf->max_degraded = 1;
4502         conf->algorithm = mddev->new_layout;
4503         conf->max_nr_stripes = NR_STRIPES;
4504         conf->reshape_progress = mddev->reshape_position;
4505         if (conf->reshape_progress != MaxSector) {
4506                 conf->prev_chunk_sectors = mddev->chunk_sectors;
4507                 conf->prev_algo = mddev->layout;
4508         }
4509
4510         memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
4511                  max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
4512         if (grow_stripes(conf, conf->max_nr_stripes)) {
4513                 printk(KERN_ERR
4514                        "md/raid:%s: couldn't allocate %dkB for buffers\n",
4515                        mdname(mddev), memory);
4516                 goto abort;
4517         } else
4518                 printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
4519                        mdname(mddev), memory);
4520
4521         conf->thread = md_register_thread(raid5d, mddev, NULL);
4522         if (!conf->thread) {
4523                 printk(KERN_ERR
4524                        "md/raid:%s: couldn't allocate thread.\n",
4525                        mdname(mddev));
4526                 goto abort;
4527         }
4528
4529         return conf;
4530
4531  abort:
4532         if (conf) {
4533                 free_conf(conf);
4534                 return ERR_PTR(-EIO);
4535         } else
4536                 return ERR_PTR(-ENOMEM);
4537 }
4538
4539
4540 static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
4541 {
4542         switch (algo) {
4543         case ALGORITHM_PARITY_0:
4544                 if (raid_disk < max_degraded)
4545                         return 1;
4546                 break;
4547         case ALGORITHM_PARITY_N:
4548                 if (raid_disk >= raid_disks - max_degraded)
4549                         return 1;
4550                 break;
4551         case ALGORITHM_PARITY_0_6:
4552                 if (raid_disk == 0 || 
4553                     raid_disk == raid_disks - 1)
4554                         return 1;
4555                 break;
4556         case ALGORITHM_LEFT_ASYMMETRIC_6:
4557         case ALGORITHM_RIGHT_ASYMMETRIC_6:
4558         case ALGORITHM_LEFT_SYMMETRIC_6:
4559         case ALGORITHM_RIGHT_SYMMETRIC_6:
4560                 if (raid_disk == raid_disks - 1)
4561                         return 1;
4562         }
4563         return 0;
4564 }
4565
4566 static int run(mddev_t *mddev)
4567 {
4568         raid5_conf_t *conf;
4569         int working_disks = 0;
4570         int dirty_parity_disks = 0;
4571         mdk_rdev_t *rdev;
4572         sector_t reshape_offset = 0;
4573
4574         if (mddev->recovery_cp != MaxSector)
4575                 printk(KERN_NOTICE "md/raid:%s: not clean"
4576                        " -- starting background reconstruction\n",
4577                        mdname(mddev));
4578         if (mddev->reshape_position != MaxSector) {
4579                 /* Check that we can continue the reshape.
4580                  * Currently only disks can change, it must
4581                  * increase, and we must be past the point where
4582                  * a stripe over-writes itself
4583                  */
4584                 sector_t here_new, here_old;
4585                 int old_disks;
4586                 int max_degraded = (mddev->level == 6 ? 2 : 1);
4587
4588                 if (mddev->new_level != mddev->level) {
4589                         printk(KERN_ERR "md/raid:%s: unsupported reshape "
4590                                "required - aborting.\n",
4591                                mdname(mddev));
4592                         return -EINVAL;
4593                 }
4594                 old_disks = mddev->raid_disks - mddev->delta_disks;
4595                 /* reshape_position must be on a new-stripe boundary, and one
4596                  * further up in new geometry must map after here in old
4597                  * geometry.
4598                  */
4599                 here_new = mddev->reshape_position;
4600                 if (sector_div(here_new, mddev->new_chunk_sectors *
4601                                (mddev->raid_disks - max_degraded))) {
4602                         printk(KERN_ERR "md/raid:%s: reshape_position not "
4603                                "on a stripe boundary\n", mdname(mddev));
4604                         return -EINVAL;
4605                 }
4606                 reshape_offset = here_new * mddev->new_chunk_sectors;
4607                 /* here_new is the stripe we will write to */
4608                 here_old = mddev->reshape_position;
4609                 sector_div(here_old, mddev->chunk_sectors *
4610                            (old_disks-max_degraded));
4611                 /* here_old is the first stripe that we might need to read
4612                  * from */
4613                 if (mddev->delta_disks == 0) {
4614                         /* We cannot be sure it is safe to start an in-place
4615                          * reshape.  It is only safe if user-space if monitoring
4616                          * and taking constant backups.
4617                          * mdadm always starts a situation like this in
4618                          * readonly mode so it can take control before
4619                          * allowing any writes.  So just check for that.
4620                          */
4621                         if ((here_new * mddev->new_chunk_sectors != 
4622                              here_old * mddev->chunk_sectors) ||
4623                             mddev->ro == 0) {
4624                                 printk(KERN_ERR "md/raid:%s: in-place reshape must be started"
4625                                        " in read-only mode - aborting\n",
4626                                        mdname(mddev));
4627                                 return -EINVAL;
4628                         }
4629                 } else if (mddev->delta_disks < 0
4630                     ? (here_new * mddev->new_chunk_sectors <=
4631                        here_old * mddev->chunk_sectors)
4632                     : (here_new * mddev->new_chunk_sectors >=
4633                        here_old * mddev->chunk_sectors)) {
4634                         /* Reading from the same stripe as writing to - bad */
4635                         printk(KERN_ERR "md/raid:%s: reshape_position too early for "
4636                                "auto-recovery - aborting.\n",
4637                                mdname(mddev));
4638                         return -EINVAL;
4639                 }
4640                 printk(KERN_INFO "md/raid:%s: reshape will continue\n",
4641                        mdname(mddev));
4642                 /* OK, we should be able to continue; */
4643         } else {
4644                 BUG_ON(mddev->level != mddev->new_level);
4645                 BUG_ON(mddev->layout != mddev->new_layout);
4646                 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
4647                 BUG_ON(mddev->delta_disks != 0);
4648         }
4649
4650         if (mddev->private == NULL)
4651                 conf = setup_conf(mddev);
4652         else
4653                 conf = mddev->private;
4654
4655         if (IS_ERR(conf))
4656                 return PTR_ERR(conf);
4657
4658         mddev->thread = conf->thread;
4659         conf->thread = NULL;
4660         mddev->private = conf;
4661
4662         /*
4663          * 0 for a fully functional array, 1 or 2 for a degraded array.
4664          */
4665         list_for_each_entry(rdev, &mddev->disks, same_set) {
4666                 if (rdev->raid_disk < 0)
4667                         continue;
4668                 if (test_bit(In_sync, &rdev->flags)) {
4669                         working_disks++;
4670                         continue;
4671                 }
4672                 /* This disc is not fully in-sync.  However if it
4673                  * just stored parity (beyond the recovery_offset),
4674                  * when we don't need to be concerned about the
4675                  * array being dirty.
4676                  * When reshape goes 'backwards', we never have
4677                  * partially completed devices, so we only need
4678                  * to worry about reshape going forwards.
4679                  */
4680                 /* Hack because v0.91 doesn't store recovery_offset properly. */
4681                 if (mddev->major_version == 0 &&
4682                     mddev->minor_version > 90)
4683                         rdev->recovery_offset = reshape_offset;
4684                         
4685                 if (rdev->recovery_offset < reshape_offset) {
4686                         /* We need to check old and new layout */
4687                         if (!only_parity(rdev->raid_disk,
4688                                          conf->algorithm,
4689                                          conf->raid_disks,
4690                                          conf->max_degraded))
4691                                 continue;
4692                 }
4693                 if (!only_parity(rdev->raid_disk,
4694                                  conf->prev_algo,
4695                                  conf->previous_raid_disks,
4696                                  conf->max_degraded))
4697                         continue;
4698                 dirty_parity_disks++;
4699         }
4700
4701         mddev->degraded = (max(conf->raid_disks, conf->previous_raid_disks)
4702                            - working_disks);
4703
4704         if (has_failed(conf)) {
4705                 printk(KERN_ERR "md/raid:%s: not enough operational devices"
4706                         " (%d/%d failed)\n",
4707                         mdname(mddev), mddev->degraded, conf->raid_disks);
4708                 goto abort;
4709         }
4710
4711         /* device size must be a multiple of chunk size */
4712         mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
4713         mddev->resync_max_sectors = mddev->dev_sectors;
4714
4715         if (mddev->degraded > dirty_parity_disks &&
4716             mddev->recovery_cp != MaxSector) {
4717                 if (mddev->ok_start_degraded)
4718                         printk(KERN_WARNING
4719                                "md/raid:%s: starting dirty degraded array"
4720                                " - data corruption possible.\n",
4721                                mdname(mddev));
4722                 else {
4723                         printk(KERN_ERR
4724                                "md/raid:%s: cannot start dirty degraded array.\n",
4725                                mdname(mddev));
4726                         goto abort;
4727                 }
4728         }
4729
4730         if (mddev->degraded == 0)
4731                 printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
4732                        " devices, algorithm %d\n", mdname(mddev), conf->level,
4733                        mddev->raid_disks-mddev->degraded, mddev->raid_disks,
4734                        mddev->new_layout);
4735         else
4736                 printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
4737                        " out of %d devices, algorithm %d\n",
4738                        mdname(mddev), conf->level,
4739                        mddev->raid_disks - mddev->degraded,
4740                        mddev->raid_disks, mddev->new_layout);
4741
4742         print_raid5_conf(conf);
4743
4744         if (conf->reshape_progress != MaxSector) {
4745                 conf->reshape_safe = conf->reshape_progress;
4746                 atomic_set(&conf->reshape_stripes, 0);
4747                 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
4748                 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
4749                 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
4750                 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
4751                 mddev->sync_thread = md_register_thread(md_do_sync, mddev,
4752                                                         "reshape");
4753         }
4754
4755
4756         /* Ok, everything is just fine now */
4757         if (mddev->to_remove == &raid5_attrs_group)
4758                 mddev->to_remove = NULL;
4759         else if (mddev->kobj.sd &&
4760             sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
4761                 printk(KERN_WARNING
4762                        "raid5: failed to create sysfs attributes for %s\n",
4763                        mdname(mddev));
4764         md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
4765
4766         if (mddev->queue) {
4767                 int chunk_size;
4768                 /* read-ahead size must cover two whole stripes, which
4769                  * is 2 * (datadisks) * chunksize where 'n' is the
4770                  * number of raid devices
4771                  */
4772                 int data_disks = conf->previous_raid_disks - conf->max_degraded;
4773                 int stripe = data_disks *
4774                         ((mddev->chunk_sectors << 9) / PAGE_SIZE);
4775                 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
4776                         mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
4777
4778                 blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec);
4779
4780                 mddev->queue->backing_dev_info.congested_data = mddev;
4781                 mddev->queue->backing_dev_info.congested_fn = raid5_congested;
4782
4783                 chunk_size = mddev->chunk_sectors << 9;
4784                 blk_queue_io_min(mddev->queue, chunk_size);
4785                 blk_queue_io_opt(mddev->queue, chunk_size *
4786                                  (conf->raid_disks - conf->max_degraded));
4787
4788                 list_for_each_entry(rdev, &mddev->disks, same_set)
4789                         disk_stack_limits(mddev->gendisk, rdev->bdev,
4790                                           rdev->data_offset << 9);
4791         }
4792
4793         return 0;
4794 abort:
4795         md_unregister_thread(mddev->thread);
4796         mddev->thread = NULL;
4797         if (conf) {
4798                 print_raid5_conf(conf);
4799                 free_conf(conf);
4800         }
4801         mddev->private = NULL;
4802         printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
4803         return -EIO;
4804 }
4805
4806 static int stop(mddev_t *mddev)
4807 {
4808         raid5_conf_t *conf = mddev->private;
4809
4810         md_unregister_thread(mddev->thread);
4811         mddev->thread = NULL;
4812         if (mddev->queue)
4813                 mddev->queue->backing_dev_info.congested_fn = NULL;
4814         free_conf(conf);
4815         mddev->private = NULL;
4816         mddev->to_remove = &raid5_attrs_group;
4817         return 0;
4818 }
4819
4820 #ifdef DEBUG
4821 static void print_sh(struct seq_file *seq, struct stripe_head *sh)
4822 {
4823         int i;
4824
4825         seq_printf(seq, "sh %llu, pd_idx %d, state %ld.\n",
4826                    (unsigned long long)sh->sector, sh->pd_idx, sh->state);
4827         seq_printf(seq, "sh %llu,  count %d.\n",
4828                    (unsigned long long)sh->sector, atomic_read(&sh->count));
4829         seq_printf(seq, "sh %llu, ", (unsigned long long)sh->sector);
4830         for (i = 0; i < sh->disks; i++) {
4831                 seq_printf(seq, "(cache%d: %p %ld) ",
4832                            i, sh->dev[i].page, sh->dev[i].flags);
4833         }
4834         seq_printf(seq, "\n");
4835 }
4836
4837 static void printall(struct seq_file *seq, raid5_conf_t *conf)
4838 {
4839         struct stripe_head *sh;
4840         struct hlist_node *hn;
4841         int i;
4842
4843         spin_lock_irq(&conf->device_lock);
4844         for (i = 0; i < NR_HASH; i++) {
4845                 hlist_for_each_entry(sh, hn, &conf->stripe_hashtbl[i], hash) {
4846                         if (sh->raid_conf != conf)
4847                                 continue;
4848                         print_sh(seq, sh);
4849                 }
4850         }
4851         spin_unlock_irq(&conf->device_lock);
4852 }
4853 #endif
4854
4855 static void status(struct seq_file *seq, mddev_t *mddev)
4856 {
4857         raid5_conf_t *conf = mddev->private;
4858         int i;
4859
4860         seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
4861                 mddev->chunk_sectors / 2, mddev->layout);
4862         seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
4863         for (i = 0; i < conf->raid_disks; i++)
4864                 seq_printf (seq, "%s",
4865                                conf->disks[i].rdev &&
4866                                test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
4867         seq_printf (seq, "]");
4868 #ifdef DEBUG
4869         seq_printf (seq, "\n");
4870         printall(seq, conf);
4871 #endif
4872 }
4873
4874 static void print_raid5_conf (raid5_conf_t *conf)
4875 {
4876         int i;
4877         struct disk_info *tmp;
4878
4879         printk(KERN_DEBUG "RAID conf printout:\n");
4880         if (!conf) {
4881                 printk("(conf==NULL)\n");
4882                 return;
4883         }
4884         printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
4885                conf->raid_disks,
4886                conf->raid_disks - conf->mddev->degraded);
4887
4888         for (i = 0; i < conf->raid_disks; i++) {
4889                 char b[BDEVNAME_SIZE];
4890                 tmp = conf->disks + i;
4891                 if (tmp->rdev)
4892                         printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
4893                                i, !test_bit(Faulty, &tmp->rdev->flags),
4894                                bdevname(tmp->rdev->bdev, b));
4895         }
4896 }
4897
4898 static int raid5_spare_active(mddev_t *mddev)
4899 {
4900         int i;
4901         raid5_conf_t *conf = mddev->private;
4902         struct disk_info *tmp;
4903         int count = 0;
4904         unsigned long flags;
4905
4906         for (i = 0; i < conf->raid_disks; i++) {
4907                 tmp = conf->disks + i;
4908                 if (tmp->rdev
4909                     && tmp->rdev->recovery_offset == MaxSector
4910                     && !test_bit(Faulty, &tmp->rdev->flags)
4911                     && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
4912                         count++;
4913                         sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
4914                 }
4915         }
4916         spin_lock_irqsave(&conf->device_lock, flags);
4917         mddev->degraded -= count;
4918         spin_unlock_irqrestore(&conf->device_lock, flags);
4919         print_raid5_conf(conf);
4920         return count;
4921 }
4922
4923 static int raid5_remove_disk(mddev_t *mddev, int number)
4924 {
4925         raid5_conf_t *conf = mddev->private;
4926         int err = 0;
4927         mdk_rdev_t *rdev;
4928         struct disk_info *p = conf->disks + number;
4929
4930         print_raid5_conf(conf);
4931         rdev = p->rdev;
4932         if (rdev) {
4933                 if (number >= conf->raid_disks &&
4934                     conf->reshape_progress == MaxSector)
4935                         clear_bit(In_sync, &rdev->flags);
4936
4937                 if (test_bit(In_sync, &rdev->flags) ||
4938                     atomic_read(&rdev->nr_pending)) {
4939                         err = -EBUSY;
4940                         goto abort;
4941                 }
4942                 /* Only remove non-faulty devices if recovery
4943                  * isn't possible.
4944                  */
4945                 if (!test_bit(Faulty, &rdev->flags) &&
4946                     !has_failed(conf) &&
4947                     number < conf->raid_disks) {
4948                         err = -EBUSY;
4949                         goto abort;
4950                 }
4951                 p->rdev = NULL;
4952                 synchronize_rcu();
4953                 if (atomic_read(&rdev->nr_pending)) {
4954                         /* lost the race, try later */
4955                         err = -EBUSY;
4956                         p->rdev = rdev;
4957                 }
4958         }
4959 abort:
4960
4961         print_raid5_conf(conf);
4962         return err;
4963 }
4964
4965 static int raid5_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
4966 {
4967         raid5_conf_t *conf = mddev->private;
4968         int err = -EEXIST;
4969         int disk;
4970         struct disk_info *p;
4971         int first = 0;
4972         int last = conf->raid_disks - 1;
4973
4974         if (has_failed(conf))
4975                 /* no point adding a device */
4976                 return -EINVAL;
4977
4978         if (rdev->raid_disk >= 0)
4979                 first = last = rdev->raid_disk;
4980
4981         /*
4982          * find the disk ... but prefer rdev->saved_raid_disk
4983          * if possible.
4984          */
4985         if (rdev->saved_raid_disk >= 0 &&
4986             rdev->saved_raid_disk >= first &&
4987             conf->disks[rdev->saved_raid_disk].rdev == NULL)
4988                 disk = rdev->saved_raid_disk;
4989         else
4990                 disk = first;
4991         for ( ; disk <= last ; disk++)
4992                 if ((p=conf->disks + disk)->rdev == NULL) {
4993                         clear_bit(In_sync, &rdev->flags);
4994                         rdev->raid_disk = disk;
4995                         err = 0;
4996                         if (rdev->saved_raid_disk != disk)
4997                                 conf->fullsync = 1;
4998                         rcu_assign_pointer(p->rdev, rdev);
4999                         break;
5000                 }
5001         print_raid5_conf(conf);
5002         return err;
5003 }
5004
5005 static int raid5_resize(mddev_t *mddev, sector_t sectors)
5006 {
5007         /* no resync is happening, and there is enough space
5008          * on all devices, so we can resize.
5009          * We need to make sure resync covers any new space.
5010          * If the array is shrinking we should possibly wait until
5011          * any io in the removed space completes, but it hardly seems
5012          * worth it.
5013          */
5014         sectors &= ~((sector_t)mddev->chunk_sectors - 1);
5015         md_set_array_sectors(mddev, raid5_size(mddev, sectors,
5016                                                mddev->raid_disks));
5017         if (mddev->array_sectors >
5018             raid5_size(mddev, sectors, mddev->raid_disks))
5019                 return -EINVAL;
5020         set_capacity(mddev->gendisk, mddev->array_sectors);
5021         revalidate_disk(mddev->gendisk);
5022         if (sectors > mddev->dev_sectors &&
5023             mddev->recovery_cp > mddev->dev_sectors) {
5024                 mddev->recovery_cp = mddev->dev_sectors;
5025                 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
5026         }
5027         mddev->dev_sectors = sectors;
5028         mddev->resync_max_sectors = sectors;
5029         return 0;
5030 }
5031
5032 static int check_stripe_cache(mddev_t *mddev)
5033 {
5034         /* Can only proceed if there are plenty of stripe_heads.
5035          * We need a minimum of one full stripe,, and for sensible progress
5036          * it is best to have about 4 times that.
5037          * If we require 4 times, then the default 256 4K stripe_heads will
5038          * allow for chunk sizes up to 256K, which is probably OK.
5039          * If the chunk size is greater, user-space should request more
5040          * stripe_heads first.
5041          */
5042         raid5_conf_t *conf = mddev->private;
5043         if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
5044             > conf->max_nr_stripes ||
5045             ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
5046             > conf->max_nr_stripes) {
5047                 printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes.  Needed %lu\n",
5048                        mdname(mddev),
5049                        ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
5050                         / STRIPE_SIZE)*4);
5051                 return 0;
5052         }
5053         return 1;
5054 }
5055
5056 static int check_reshape(mddev_t *mddev)
5057 {
5058         raid5_conf_t *conf = mddev->private;
5059
5060         if (mddev->delta_disks == 0 &&
5061             mddev->new_layout == mddev->layout &&
5062             mddev->new_chunk_sectors == mddev->chunk_sectors)
5063                 return 0; /* nothing to do */
5064         if (mddev->bitmap)
5065                 /* Cannot grow a bitmap yet */
5066                 return -EBUSY;
5067         if (has_failed(conf))
5068                 return -EINVAL;
5069         if (mddev->delta_disks < 0) {
5070                 /* We might be able to shrink, but the devices must
5071                  * be made bigger first.
5072                  * For raid6, 4 is the minimum size.
5073                  * Otherwise 2 is the minimum
5074                  */
5075                 int min = 2;
5076                 if (mddev->level == 6)
5077                         min = 4;
5078                 if (mddev->raid_disks + mddev->delta_disks < min)
5079                         return -EINVAL;
5080         }
5081
5082         if (!check_stripe_cache(mddev))
5083                 return -ENOSPC;
5084
5085         return resize_stripes(conf, conf->raid_disks + mddev->delta_disks);
5086 }
5087
5088 static int raid5_start_reshape(mddev_t *mddev)
5089 {
5090         raid5_conf_t *conf = mddev->private;
5091         mdk_rdev_t *rdev;
5092         int spares = 0;
5093         unsigned long flags;
5094
5095         if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
5096                 return -EBUSY;
5097
5098         if (!check_stripe_cache(mddev))
5099                 return -ENOSPC;
5100
5101         list_for_each_entry(rdev, &mddev->disks, same_set)
5102                 if (!test_bit(In_sync, &rdev->flags)
5103                     && !test_bit(Faulty, &rdev->flags))
5104                         spares++;
5105
5106         if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
5107                 /* Not enough devices even to make a degraded array
5108                  * of that size
5109                  */
5110                 return -EINVAL;
5111
5112         /* Refuse to reduce size of the array.  Any reductions in
5113          * array size must be through explicit setting of array_size
5114          * attribute.
5115          */
5116         if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
5117             < mddev->array_sectors) {
5118                 printk(KERN_ERR "md/raid:%s: array size must be reduced "
5119                        "before number of disks\n", mdname(mddev));
5120                 return -EINVAL;
5121         }
5122
5123         atomic_set(&conf->reshape_stripes, 0);
5124         spin_lock_irq(&conf->device_lock);
5125         conf->previous_raid_disks = conf->raid_disks;
5126         conf->raid_disks += mddev->delta_disks;
5127         conf->prev_chunk_sectors = conf->chunk_sectors;
5128         conf->chunk_sectors = mddev->new_chunk_sectors;
5129         conf->prev_algo = conf->algorithm;
5130         conf->algorithm = mddev->new_layout;
5131         if (mddev->delta_disks < 0)
5132                 conf->reshape_progress = raid5_size(mddev, 0, 0);
5133         else
5134                 conf->reshape_progress = 0;
5135         conf->reshape_safe = conf->reshape_progress;
5136         conf->generation++;
5137         spin_unlock_irq(&conf->device_lock);
5138
5139         /* Add some new drives, as many as will fit.
5140          * We know there are enough to make the newly sized array work.
5141          * Don't add devices if we are reducing the number of
5142          * devices in the array.  This is because it is not possible
5143          * to correctly record the "partially reconstructed" state of
5144          * such devices during the reshape and confusion could result.
5145          */
5146         if (mddev->delta_disks >= 0) {
5147                 int added_devices = 0;
5148                 list_for_each_entry(rdev, &mddev->disks, same_set)
5149                         if (rdev->raid_disk < 0 &&
5150                             !test_bit(Faulty, &rdev->flags)) {
5151                                 if (raid5_add_disk(mddev, rdev) == 0) {
5152                                         if (rdev->raid_disk
5153                                             >= conf->previous_raid_disks) {
5154                                                 set_bit(In_sync, &rdev->flags);
5155                                                 added_devices++;
5156                                         } else
5157                                                 rdev->recovery_offset = 0;
5158
5159                                         if (sysfs_link_rdev(mddev, rdev))
5160                                                 /* Failure here is OK */;
5161                                 }
5162                         } else if (rdev->raid_disk >= conf->previous_raid_disks
5163                                    && !test_bit(Faulty, &rdev->flags)) {
5164                                 /* This is a spare that was manually added */
5165                                 set_bit(In_sync, &rdev->flags);
5166                                 added_devices++;
5167                         }
5168
5169                 /* When a reshape changes the number of devices,
5170                  * ->degraded is measured against the larger of the
5171                  * pre and post number of devices.
5172                  */
5173                 spin_lock_irqsave(&conf->device_lock, flags);
5174                 mddev->degraded += (conf->raid_disks - conf->previous_raid_disks)
5175                         - added_devices;
5176                 spin_unlock_irqrestore(&conf->device_lock, flags);
5177         }
5178         mddev->raid_disks = conf->raid_disks;
5179         mddev->reshape_position = conf->reshape_progress;
5180         set_bit(MD_CHANGE_DEVS, &mddev->flags);
5181
5182         clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
5183         clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
5184         set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
5185         set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
5186         mddev->sync_thread = md_register_thread(md_do_sync, mddev,
5187                                                 "reshape");
5188         if (!mddev->sync_thread) {
5189                 mddev->recovery = 0;
5190                 spin_lock_irq(&conf->device_lock);
5191                 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
5192                 conf->reshape_progress = MaxSector;
5193                 spin_unlock_irq(&conf->device_lock);
5194                 return -EAGAIN;
5195         }
5196         conf->reshape_checkpoint = jiffies;
5197         md_wakeup_thread(mddev->sync_thread);
5198         md_new_event(mddev);
5199         return 0;
5200 }
5201
5202 /* This is called from the reshape thread and should make any
5203  * changes needed in 'conf'
5204  */
5205 static void end_reshape(raid5_conf_t *conf)
5206 {
5207
5208         if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
5209
5210                 spin_lock_irq(&conf->device_lock);
5211                 conf->previous_raid_disks = conf->raid_disks;
5212                 conf->reshape_progress = MaxSector;
5213                 spin_unlock_irq(&conf->device_lock);
5214                 wake_up(&conf->wait_for_overlap);
5215
5216                 /* read-ahead size must cover two whole stripes, which is
5217                  * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
5218                  */
5219                 if (conf->mddev->queue) {
5220                         int data_disks = conf->raid_disks - conf->max_degraded;
5221                         int stripe = data_disks * ((conf->chunk_sectors << 9)
5222                                                    / PAGE_SIZE);
5223                         if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
5224                                 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
5225                 }
5226         }
5227 }
5228
5229 /* This is called from the raid5d thread with mddev_lock held.
5230  * It makes config changes to the device.
5231  */
5232 static void raid5_finish_reshape(mddev_t *mddev)
5233 {
5234         raid5_conf_t *conf = mddev->private;
5235
5236         if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
5237
5238                 if (mddev->delta_disks > 0) {
5239                         md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
5240                         set_capacity(mddev->gendisk, mddev->array_sectors);
5241                         revalidate_disk(mddev->gendisk);
5242                 } else {
5243                         int d;
5244                         mddev->degraded = conf->raid_disks;
5245                         for (d = 0; d < conf->raid_disks ; d++)
5246                                 if (conf->disks[d].rdev &&
5247                                     test_bit(In_sync,
5248                                              &conf->disks[d].rdev->flags))
5249                                         mddev->degraded--;
5250                         for (d = conf->raid_disks ;
5251                              d < conf->raid_disks - mddev->delta_disks;
5252                              d++) {
5253                                 mdk_rdev_t *rdev = conf->disks[d].rdev;
5254                                 if (rdev && raid5_remove_disk(mddev, d) == 0) {
5255                                         sysfs_unlink_rdev(mddev, rdev);
5256                                         rdev->raid_disk = -1;
5257                                 }
5258                         }
5259                 }
5260                 mddev->layout = conf->algorithm;
5261                 mddev->chunk_sectors = conf->chunk_sectors;
5262                 mddev->reshape_position = MaxSector;
5263                 mddev->delta_disks = 0;
5264         }
5265 }
5266
5267 static void raid5_quiesce(mddev_t *mddev, int state)
5268 {
5269         raid5_conf_t *conf = mddev->private;
5270
5271         switch(state) {
5272         case 2: /* resume for a suspend */
5273                 wake_up(&conf->wait_for_overlap);
5274                 break;
5275
5276         case 1: /* stop all writes */
5277                 spin_lock_irq(&conf->device_lock);
5278                 /* '2' tells resync/reshape to pause so that all
5279                  * active stripes can drain
5280                  */
5281                 conf->quiesce = 2;
5282                 wait_event_lock_irq(conf->wait_for_stripe,
5283                                     atomic_read(&conf->active_stripes) == 0 &&
5284                                     atomic_read(&conf->active_aligned_reads) == 0,
5285                                     conf->device_lock, /* nothing */);
5286                 conf->quiesce = 1;
5287                 spin_unlock_irq(&conf->device_lock);
5288                 /* allow reshape to continue */
5289                 wake_up(&conf->wait_for_overlap);
5290                 break;
5291
5292         case 0: /* re-enable writes */
5293                 spin_lock_irq(&conf->device_lock);
5294                 conf->quiesce = 0;
5295                 wake_up(&conf->wait_for_stripe);
5296                 wake_up(&conf->wait_for_overlap);
5297                 spin_unlock_irq(&conf->device_lock);
5298                 break;
5299         }
5300 }
5301
5302
5303 static void *raid45_takeover_raid0(mddev_t *mddev, int level)
5304 {
5305         struct raid0_private_data *raid0_priv = mddev->private;
5306         sector_t sectors;
5307
5308         /* for raid0 takeover only one zone is supported */
5309         if (raid0_priv->nr_strip_zones > 1) {
5310                 printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
5311                        mdname(mddev));
5312                 return ERR_PTR(-EINVAL);
5313         }
5314
5315         sectors = raid0_priv->strip_zone[0].zone_end;
5316         sector_div(sectors, raid0_priv->strip_zone[0].nb_dev);
5317         mddev->dev_sectors = sectors;
5318         mddev->new_level = level;
5319         mddev->new_layout = ALGORITHM_PARITY_N;
5320         mddev->new_chunk_sectors = mddev->chunk_sectors;
5321         mddev->raid_disks += 1;
5322         mddev->delta_disks = 1;
5323         /* make sure it will be not marked as dirty */
5324         mddev->recovery_cp = MaxSector;
5325
5326         return setup_conf(mddev);
5327 }
5328
5329
5330 static void *raid5_takeover_raid1(mddev_t *mddev)
5331 {
5332         int chunksect;
5333
5334         if (mddev->raid_disks != 2 ||
5335             mddev->degraded > 1)
5336                 return ERR_PTR(-EINVAL);
5337
5338         /* Should check if there are write-behind devices? */
5339
5340         chunksect = 64*2; /* 64K by default */
5341
5342         /* The array must be an exact multiple of chunksize */
5343         while (chunksect && (mddev->array_sectors & (chunksect-1)))
5344                 chunksect >>= 1;
5345
5346         if ((chunksect<<9) < STRIPE_SIZE)
5347                 /* array size does not allow a suitable chunk size */
5348                 return ERR_PTR(-EINVAL);
5349
5350         mddev->new_level = 5;
5351         mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
5352         mddev->new_chunk_sectors = chunksect;
5353
5354         return setup_conf(mddev);
5355 }
5356
5357 static void *raid5_takeover_raid6(mddev_t *mddev)
5358 {
5359         int new_layout;
5360
5361         switch (mddev->layout) {
5362         case ALGORITHM_LEFT_ASYMMETRIC_6:
5363                 new_layout = ALGORITHM_LEFT_ASYMMETRIC;
5364                 break;
5365         case ALGORITHM_RIGHT_ASYMMETRIC_6:
5366                 new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
5367                 break;
5368         case ALGORITHM_LEFT_SYMMETRIC_6:
5369                 new_layout = ALGORITHM_LEFT_SYMMETRIC;
5370                 break;
5371         case ALGORITHM_RIGHT_SYMMETRIC_6:
5372                 new_layout = ALGORITHM_RIGHT_SYMMETRIC;
5373