01505406ca7b4ac69b95a925a8f73494e8c0d9ac
[linux-3.10.git] / drivers / mtd / nand / nand_base.c
1 /*
2  *  drivers/mtd/nand.c
3  *
4  *  Overview:
5  *   This is the generic MTD driver for NAND flash devices. It should be
6  *   capable of working with almost all NAND chips currently available.
7  *   Basic support for AG-AND chips is provided.
8  *
9  *      Additional technical information is available on
10  *      http://www.linux-mtd.infradead.org/doc/nand.html
11  *
12  *  Copyright (C) 2000 Steven J. Hill (sjhill@realitydiluted.com)
13  *                2002-2006 Thomas Gleixner (tglx@linutronix.de)
14  *
15  *  Credits:
16  *      David Woodhouse for adding multichip support
17  *
18  *      Aleph One Ltd. and Toby Churchill Ltd. for supporting the
19  *      rework for 2K page size chips
20  *
21  *  TODO:
22  *      Enable cached programming for 2k page size chips
23  *      Check, if mtd->ecctype should be set to MTD_ECC_HW
24  *      if we have HW ECC support.
25  *      The AG-AND chips have nice features for speed improvement,
26  *      which are not supported yet. Read / program 4 pages in one go.
27  *      BBT table is not serialized, has to be fixed
28  *
29  * This program is free software; you can redistribute it and/or modify
30  * it under the terms of the GNU General Public License version 2 as
31  * published by the Free Software Foundation.
32  *
33  */
34
35 #include <linux/module.h>
36 #include <linux/delay.h>
37 #include <linux/errno.h>
38 #include <linux/err.h>
39 #include <linux/sched.h>
40 #include <linux/slab.h>
41 #include <linux/types.h>
42 #include <linux/mtd/mtd.h>
43 #include <linux/mtd/nand.h>
44 #include <linux/mtd/nand_ecc.h>
45 #include <linux/mtd/nand_bch.h>
46 #include <linux/interrupt.h>
47 #include <linux/bitops.h>
48 #include <linux/leds.h>
49 #include <linux/io.h>
50 #include <linux/mtd/partitions.h>
51
52 /* Define default oob placement schemes for large and small page devices */
53 static struct nand_ecclayout nand_oob_8 = {
54         .eccbytes = 3,
55         .eccpos = {0, 1, 2},
56         .oobfree = {
57                 {.offset = 3,
58                  .length = 2},
59                 {.offset = 6,
60                  .length = 2} }
61 };
62
63 static struct nand_ecclayout nand_oob_16 = {
64         .eccbytes = 6,
65         .eccpos = {0, 1, 2, 3, 6, 7},
66         .oobfree = {
67                 {.offset = 8,
68                  . length = 8} }
69 };
70
71 static struct nand_ecclayout nand_oob_64 = {
72         .eccbytes = 24,
73         .eccpos = {
74                    40, 41, 42, 43, 44, 45, 46, 47,
75                    48, 49, 50, 51, 52, 53, 54, 55,
76                    56, 57, 58, 59, 60, 61, 62, 63},
77         .oobfree = {
78                 {.offset = 2,
79                  .length = 38} }
80 };
81
82 static struct nand_ecclayout nand_oob_128 = {
83         .eccbytes = 48,
84         .eccpos = {
85                    80, 81, 82, 83, 84, 85, 86, 87,
86                    88, 89, 90, 91, 92, 93, 94, 95,
87                    96, 97, 98, 99, 100, 101, 102, 103,
88                    104, 105, 106, 107, 108, 109, 110, 111,
89                    112, 113, 114, 115, 116, 117, 118, 119,
90                    120, 121, 122, 123, 124, 125, 126, 127},
91         .oobfree = {
92                 {.offset = 2,
93                  .length = 78} }
94 };
95
96 static int nand_get_device(struct nand_chip *chip, struct mtd_info *mtd,
97                            int new_state);
98
99 static int nand_do_write_oob(struct mtd_info *mtd, loff_t to,
100                              struct mtd_oob_ops *ops);
101
102 /*
103  * For devices which display every fart in the system on a separate LED. Is
104  * compiled away when LED support is disabled.
105  */
106 DEFINE_LED_TRIGGER(nand_led_trigger);
107
108 static int check_offs_len(struct mtd_info *mtd,
109                                         loff_t ofs, uint64_t len)
110 {
111         struct nand_chip *chip = mtd->priv;
112         int ret = 0;
113
114         /* Start address must align on block boundary */
115         if (ofs & ((1 << chip->phys_erase_shift) - 1)) {
116                 pr_debug("%s: unaligned address\n", __func__);
117                 ret = -EINVAL;
118         }
119
120         /* Length must align on block boundary */
121         if (len & ((1 << chip->phys_erase_shift) - 1)) {
122                 pr_debug("%s: length not block aligned\n", __func__);
123                 ret = -EINVAL;
124         }
125
126         return ret;
127 }
128
129 /**
130  * nand_release_device - [GENERIC] release chip
131  * @mtd: MTD device structure
132  *
133  * Deselect, release chip lock and wake up anyone waiting on the device.
134  */
135 static void nand_release_device(struct mtd_info *mtd)
136 {
137         struct nand_chip *chip = mtd->priv;
138
139         /* De-select the NAND device */
140         chip->select_chip(mtd, -1);
141
142         /* Release the controller and the chip */
143         spin_lock(&chip->controller->lock);
144         chip->controller->active = NULL;
145         chip->state = FL_READY;
146         wake_up(&chip->controller->wq);
147         spin_unlock(&chip->controller->lock);
148 }
149
150 /**
151  * nand_read_byte - [DEFAULT] read one byte from the chip
152  * @mtd: MTD device structure
153  *
154  * Default read function for 8bit buswidth
155  */
156 static uint8_t nand_read_byte(struct mtd_info *mtd)
157 {
158         struct nand_chip *chip = mtd->priv;
159         return readb(chip->IO_ADDR_R);
160 }
161
162 /**
163  * nand_read_byte16 - [DEFAULT] read one byte endianess aware from the chip
164  * nand_read_byte16 - [DEFAULT] read one byte endianness aware from the chip
165  * @mtd: MTD device structure
166  *
167  * Default read function for 16bit buswidth with endianness conversion.
168  *
169  */
170 static uint8_t nand_read_byte16(struct mtd_info *mtd)
171 {
172         struct nand_chip *chip = mtd->priv;
173         return (uint8_t) cpu_to_le16(readw(chip->IO_ADDR_R));
174 }
175
176 /**
177  * nand_read_word - [DEFAULT] read one word from the chip
178  * @mtd: MTD device structure
179  *
180  * Default read function for 16bit buswidth without endianness conversion.
181  */
182 static u16 nand_read_word(struct mtd_info *mtd)
183 {
184         struct nand_chip *chip = mtd->priv;
185         return readw(chip->IO_ADDR_R);
186 }
187
188 /**
189  * nand_select_chip - [DEFAULT] control CE line
190  * @mtd: MTD device structure
191  * @chipnr: chipnumber to select, -1 for deselect
192  *
193  * Default select function for 1 chip devices.
194  */
195 static void nand_select_chip(struct mtd_info *mtd, int chipnr)
196 {
197         struct nand_chip *chip = mtd->priv;
198
199         switch (chipnr) {
200         case -1:
201                 chip->cmd_ctrl(mtd, NAND_CMD_NONE, 0 | NAND_CTRL_CHANGE);
202                 break;
203         case 0:
204                 break;
205
206         default:
207                 BUG();
208         }
209 }
210
211 /**
212  * nand_write_buf - [DEFAULT] write buffer to chip
213  * @mtd: MTD device structure
214  * @buf: data buffer
215  * @len: number of bytes to write
216  *
217  * Default write function for 8bit buswidth.
218  */
219 static void nand_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len)
220 {
221         int i;
222         struct nand_chip *chip = mtd->priv;
223
224         for (i = 0; i < len; i++)
225                 writeb(buf[i], chip->IO_ADDR_W);
226 }
227
228 /**
229  * nand_read_buf - [DEFAULT] read chip data into buffer
230  * @mtd: MTD device structure
231  * @buf: buffer to store date
232  * @len: number of bytes to read
233  *
234  * Default read function for 8bit buswidth.
235  */
236 static void nand_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
237 {
238         int i;
239         struct nand_chip *chip = mtd->priv;
240
241         for (i = 0; i < len; i++)
242                 buf[i] = readb(chip->IO_ADDR_R);
243 }
244
245 /**
246  * nand_write_buf16 - [DEFAULT] write buffer to chip
247  * @mtd: MTD device structure
248  * @buf: data buffer
249  * @len: number of bytes to write
250  *
251  * Default write function for 16bit buswidth.
252  */
253 static void nand_write_buf16(struct mtd_info *mtd, const uint8_t *buf, int len)
254 {
255         int i;
256         struct nand_chip *chip = mtd->priv;
257         u16 *p = (u16 *) buf;
258         len >>= 1;
259
260         for (i = 0; i < len; i++)
261                 writew(p[i], chip->IO_ADDR_W);
262
263 }
264
265 /**
266  * nand_read_buf16 - [DEFAULT] read chip data into buffer
267  * @mtd: MTD device structure
268  * @buf: buffer to store date
269  * @len: number of bytes to read
270  *
271  * Default read function for 16bit buswidth.
272  */
273 static void nand_read_buf16(struct mtd_info *mtd, uint8_t *buf, int len)
274 {
275         int i;
276         struct nand_chip *chip = mtd->priv;
277         u16 *p = (u16 *) buf;
278         len >>= 1;
279
280         for (i = 0; i < len; i++)
281                 p[i] = readw(chip->IO_ADDR_R);
282 }
283
284 /**
285  * nand_block_bad - [DEFAULT] Read bad block marker from the chip
286  * @mtd: MTD device structure
287  * @ofs: offset from device start
288  * @getchip: 0, if the chip is already selected
289  *
290  * Check, if the block is bad.
291  */
292 static int nand_block_bad(struct mtd_info *mtd, loff_t ofs, int getchip)
293 {
294         int page, chipnr, res = 0, i = 0;
295         struct nand_chip *chip = mtd->priv;
296         u16 bad;
297
298         if (chip->bbt_options & NAND_BBT_SCANLASTPAGE)
299                 ofs += mtd->erasesize - mtd->writesize;
300
301         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
302
303         if (getchip) {
304                 chipnr = (int)(ofs >> chip->chip_shift);
305
306                 nand_get_device(chip, mtd, FL_READING);
307
308                 /* Select the NAND device */
309                 chip->select_chip(mtd, chipnr);
310         }
311
312         do {
313                 if (chip->options & NAND_BUSWIDTH_16) {
314                         chip->cmdfunc(mtd, NAND_CMD_READOOB,
315                                         chip->badblockpos & 0xFE, page);
316                         bad = cpu_to_le16(chip->read_word(mtd));
317                         if (chip->badblockpos & 0x1)
318                                 bad >>= 8;
319                         else
320                                 bad &= 0xFF;
321                 } else {
322                         chip->cmdfunc(mtd, NAND_CMD_READOOB, chip->badblockpos,
323                                         page);
324                         bad = chip->read_byte(mtd);
325                 }
326
327                 if (likely(chip->badblockbits == 8))
328                         res = bad != 0xFF;
329                 else
330                         res = hweight8(bad) < chip->badblockbits;
331                 ofs += mtd->writesize;
332                 page = (int)(ofs >> chip->page_shift) & chip->pagemask;
333                 i++;
334         } while (!res && i < 2 && (chip->bbt_options & NAND_BBT_SCAN2NDPAGE));
335
336         if (getchip)
337                 nand_release_device(mtd);
338
339         return res;
340 }
341
342 /**
343  * nand_default_block_markbad - [DEFAULT] mark a block bad
344  * @mtd: MTD device structure
345  * @ofs: offset from device start
346  *
347  * This is the default implementation, which can be overridden by a hardware
348  * specific driver. We try operations in the following order, according to our
349  * bbt_options (NAND_BBT_NO_OOB_BBM and NAND_BBT_USE_FLASH):
350  *  (1) erase the affected block, to allow OOB marker to be written cleanly
351  *  (2) update in-memory BBT
352  *  (3) write bad block marker to OOB area of affected block
353  *  (4) update flash-based BBT
354  * Note that we retain the first error encountered in (3) or (4), finish the
355  * procedures, and dump the error in the end.
356 */
357 static int nand_default_block_markbad(struct mtd_info *mtd, loff_t ofs)
358 {
359         struct nand_chip *chip = mtd->priv;
360         uint8_t buf[2] = { 0, 0 };
361         int block, res, ret = 0, i = 0;
362         int write_oob = !(chip->bbt_options & NAND_BBT_NO_OOB_BBM);
363
364         if (write_oob) {
365                 struct erase_info einfo;
366
367                 /* Attempt erase before marking OOB */
368                 memset(&einfo, 0, sizeof(einfo));
369                 einfo.mtd = mtd;
370                 einfo.addr = ofs;
371                 einfo.len = 1 << chip->phys_erase_shift;
372                 nand_erase_nand(mtd, &einfo, 0);
373         }
374
375         /* Get block number */
376         block = (int)(ofs >> chip->bbt_erase_shift);
377         /* Mark block bad in memory-based BBT */
378         if (chip->bbt)
379                 chip->bbt[block >> 2] |= 0x01 << ((block & 0x03) << 1);
380
381         /* Write bad block marker to OOB */
382         if (write_oob) {
383                 struct mtd_oob_ops ops;
384                 loff_t wr_ofs = ofs;
385
386                 nand_get_device(chip, mtd, FL_WRITING);
387
388                 ops.datbuf = NULL;
389                 ops.oobbuf = buf;
390                 ops.ooboffs = chip->badblockpos;
391                 if (chip->options & NAND_BUSWIDTH_16) {
392                         ops.ooboffs &= ~0x01;
393                         ops.len = ops.ooblen = 2;
394                 } else {
395                         ops.len = ops.ooblen = 1;
396                 }
397                 ops.mode = MTD_OPS_PLACE_OOB;
398
399                 /* Write to first/last page(s) if necessary */
400                 if (chip->bbt_options & NAND_BBT_SCANLASTPAGE)
401                         wr_ofs += mtd->erasesize - mtd->writesize;
402                 do {
403                         res = nand_do_write_oob(mtd, wr_ofs, &ops);
404                         if (!ret)
405                                 ret = res;
406
407                         i++;
408                         wr_ofs += mtd->writesize;
409                 } while ((chip->bbt_options & NAND_BBT_SCAN2NDPAGE) && i < 2);
410
411                 nand_release_device(mtd);
412         }
413
414         /* Update flash-based bad block table */
415         if (chip->bbt_options & NAND_BBT_USE_FLASH) {
416                 res = nand_update_bbt(mtd, ofs);
417                 if (!ret)
418                         ret = res;
419         }
420
421         if (!ret)
422                 mtd->ecc_stats.badblocks++;
423
424         return ret;
425 }
426
427 /**
428  * nand_check_wp - [GENERIC] check if the chip is write protected
429  * @mtd: MTD device structure
430  *
431  * Check, if the device is write protected. The function expects, that the
432  * device is already selected.
433  */
434 static int nand_check_wp(struct mtd_info *mtd)
435 {
436         struct nand_chip *chip = mtd->priv;
437
438         /* Broken xD cards report WP despite being writable */
439         if (chip->options & NAND_BROKEN_XD)
440                 return 0;
441
442         /* Check the WP bit */
443         chip->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1);
444         return (chip->read_byte(mtd) & NAND_STATUS_WP) ? 0 : 1;
445 }
446
447 /**
448  * nand_block_checkbad - [GENERIC] Check if a block is marked bad
449  * @mtd: MTD device structure
450  * @ofs: offset from device start
451  * @getchip: 0, if the chip is already selected
452  * @allowbbt: 1, if its allowed to access the bbt area
453  *
454  * Check, if the block is bad. Either by reading the bad block table or
455  * calling of the scan function.
456  */
457 static int nand_block_checkbad(struct mtd_info *mtd, loff_t ofs, int getchip,
458                                int allowbbt)
459 {
460         struct nand_chip *chip = mtd->priv;
461
462         if (!chip->bbt)
463                 return chip->block_bad(mtd, ofs, getchip);
464
465         /* Return info from the table */
466         return nand_isbad_bbt(mtd, ofs, allowbbt);
467 }
468
469 /**
470  * panic_nand_wait_ready - [GENERIC] Wait for the ready pin after commands.
471  * @mtd: MTD device structure
472  * @timeo: Timeout
473  *
474  * Helper function for nand_wait_ready used when needing to wait in interrupt
475  * context.
476  */
477 static void panic_nand_wait_ready(struct mtd_info *mtd, unsigned long timeo)
478 {
479         struct nand_chip *chip = mtd->priv;
480         int i;
481
482         /* Wait for the device to get ready */
483         for (i = 0; i < timeo; i++) {
484                 if (chip->dev_ready(mtd))
485                         break;
486                 touch_softlockup_watchdog();
487                 mdelay(1);
488         }
489 }
490
491 /* Wait for the ready pin, after a command. The timeout is caught later. */
492 void nand_wait_ready(struct mtd_info *mtd)
493 {
494         struct nand_chip *chip = mtd->priv;
495         unsigned long timeo = jiffies + 2;
496
497         /* 400ms timeout */
498         if (in_interrupt() || oops_in_progress)
499                 return panic_nand_wait_ready(mtd, 400);
500
501         led_trigger_event(nand_led_trigger, LED_FULL);
502         /* Wait until command is processed or timeout occurs */
503         do {
504                 if (chip->dev_ready(mtd))
505                         break;
506                 touch_softlockup_watchdog();
507         } while (time_before(jiffies, timeo));
508         led_trigger_event(nand_led_trigger, LED_OFF);
509 }
510 EXPORT_SYMBOL_GPL(nand_wait_ready);
511
512 /**
513  * nand_command - [DEFAULT] Send command to NAND device
514  * @mtd: MTD device structure
515  * @command: the command to be sent
516  * @column: the column address for this command, -1 if none
517  * @page_addr: the page address for this command, -1 if none
518  *
519  * Send command to NAND device. This function is used for small page devices
520  * (256/512 Bytes per page).
521  */
522 static void nand_command(struct mtd_info *mtd, unsigned int command,
523                          int column, int page_addr)
524 {
525         register struct nand_chip *chip = mtd->priv;
526         int ctrl = NAND_CTRL_CLE | NAND_CTRL_CHANGE;
527
528         /* Write out the command to the device */
529         if (command == NAND_CMD_SEQIN) {
530                 int readcmd;
531
532                 if (column >= mtd->writesize) {
533                         /* OOB area */
534                         column -= mtd->writesize;
535                         readcmd = NAND_CMD_READOOB;
536                 } else if (column < 256) {
537                         /* First 256 bytes --> READ0 */
538                         readcmd = NAND_CMD_READ0;
539                 } else {
540                         column -= 256;
541                         readcmd = NAND_CMD_READ1;
542                 }
543                 chip->cmd_ctrl(mtd, readcmd, ctrl);
544                 ctrl &= ~NAND_CTRL_CHANGE;
545         }
546         chip->cmd_ctrl(mtd, command, ctrl);
547
548         /* Address cycle, when necessary */
549         ctrl = NAND_CTRL_ALE | NAND_CTRL_CHANGE;
550         /* Serially input address */
551         if (column != -1) {
552                 /* Adjust columns for 16 bit buswidth */
553                 if (chip->options & NAND_BUSWIDTH_16)
554                         column >>= 1;
555                 chip->cmd_ctrl(mtd, column, ctrl);
556                 ctrl &= ~NAND_CTRL_CHANGE;
557         }
558         if (page_addr != -1) {
559                 chip->cmd_ctrl(mtd, page_addr, ctrl);
560                 ctrl &= ~NAND_CTRL_CHANGE;
561                 chip->cmd_ctrl(mtd, page_addr >> 8, ctrl);
562                 /* One more address cycle for devices > 32MiB */
563                 if (chip->chipsize > (32 << 20))
564                         chip->cmd_ctrl(mtd, page_addr >> 16, ctrl);
565         }
566         chip->cmd_ctrl(mtd, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE);
567
568         /*
569          * Program and erase have their own busy handlers status and sequential
570          * in needs no delay
571          */
572         switch (command) {
573
574         case NAND_CMD_PAGEPROG:
575         case NAND_CMD_ERASE1:
576         case NAND_CMD_ERASE2:
577         case NAND_CMD_SEQIN:
578         case NAND_CMD_STATUS:
579                 return;
580
581         case NAND_CMD_RESET:
582                 if (chip->dev_ready)
583                         break;
584                 udelay(chip->chip_delay);
585                 chip->cmd_ctrl(mtd, NAND_CMD_STATUS,
586                                NAND_CTRL_CLE | NAND_CTRL_CHANGE);
587                 chip->cmd_ctrl(mtd,
588                                NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE);
589                 while (!(chip->read_byte(mtd) & NAND_STATUS_READY))
590                                 ;
591                 return;
592
593                 /* This applies to read commands */
594         default:
595                 /*
596                  * If we don't have access to the busy pin, we apply the given
597                  * command delay
598                  */
599                 if (!chip->dev_ready) {
600                         udelay(chip->chip_delay);
601                         return;
602                 }
603         }
604         /*
605          * Apply this short delay always to ensure that we do wait tWB in
606          * any case on any machine.
607          */
608         ndelay(100);
609
610         nand_wait_ready(mtd);
611 }
612
613 /**
614  * nand_command_lp - [DEFAULT] Send command to NAND large page device
615  * @mtd: MTD device structure
616  * @command: the command to be sent
617  * @column: the column address for this command, -1 if none
618  * @page_addr: the page address for this command, -1 if none
619  *
620  * Send command to NAND device. This is the version for the new large page
621  * devices. We don't have the separate regions as we have in the small page
622  * devices. We must emulate NAND_CMD_READOOB to keep the code compatible.
623  */
624 static void nand_command_lp(struct mtd_info *mtd, unsigned int command,
625                             int column, int page_addr)
626 {
627         register struct nand_chip *chip = mtd->priv;
628
629         /* Emulate NAND_CMD_READOOB */
630         if (command == NAND_CMD_READOOB) {
631                 column += mtd->writesize;
632                 command = NAND_CMD_READ0;
633         }
634
635         /* Command latch cycle */
636         chip->cmd_ctrl(mtd, command & 0xff,
637                        NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE);
638
639         if (column != -1 || page_addr != -1) {
640                 int ctrl = NAND_CTRL_CHANGE | NAND_NCE | NAND_ALE;
641
642                 /* Serially input address */
643                 if (column != -1) {
644                         /* Adjust columns for 16 bit buswidth */
645                         if (chip->options & NAND_BUSWIDTH_16)
646                                 column >>= 1;
647                         chip->cmd_ctrl(mtd, column, ctrl);
648                         ctrl &= ~NAND_CTRL_CHANGE;
649                         chip->cmd_ctrl(mtd, column >> 8, ctrl);
650                 }
651                 if (page_addr != -1) {
652                         chip->cmd_ctrl(mtd, page_addr, ctrl);
653                         chip->cmd_ctrl(mtd, page_addr >> 8,
654                                        NAND_NCE | NAND_ALE);
655                         /* One more address cycle for devices > 128MiB */
656                         if (chip->chipsize > (128 << 20))
657                                 chip->cmd_ctrl(mtd, page_addr >> 16,
658                                                NAND_NCE | NAND_ALE);
659                 }
660         }
661         chip->cmd_ctrl(mtd, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE);
662
663         /*
664          * Program and erase have their own busy handlers status, sequential
665          * in, and deplete1 need no delay.
666          */
667         switch (command) {
668
669         case NAND_CMD_CACHEDPROG:
670         case NAND_CMD_PAGEPROG:
671         case NAND_CMD_ERASE1:
672         case NAND_CMD_ERASE2:
673         case NAND_CMD_SEQIN:
674         case NAND_CMD_RNDIN:
675         case NAND_CMD_STATUS:
676         case NAND_CMD_DEPLETE1:
677                 return;
678
679         case NAND_CMD_STATUS_ERROR:
680         case NAND_CMD_STATUS_ERROR0:
681         case NAND_CMD_STATUS_ERROR1:
682         case NAND_CMD_STATUS_ERROR2:
683         case NAND_CMD_STATUS_ERROR3:
684                 /* Read error status commands require only a short delay */
685                 udelay(chip->chip_delay);
686                 return;
687
688         case NAND_CMD_RESET:
689                 if (chip->dev_ready)
690                         break;
691                 udelay(chip->chip_delay);
692                 chip->cmd_ctrl(mtd, NAND_CMD_STATUS,
693                                NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE);
694                 chip->cmd_ctrl(mtd, NAND_CMD_NONE,
695                                NAND_NCE | NAND_CTRL_CHANGE);
696                 while (!(chip->read_byte(mtd) & NAND_STATUS_READY))
697                                 ;
698                 return;
699
700         case NAND_CMD_RNDOUT:
701                 /* No ready / busy check necessary */
702                 chip->cmd_ctrl(mtd, NAND_CMD_RNDOUTSTART,
703                                NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE);
704                 chip->cmd_ctrl(mtd, NAND_CMD_NONE,
705                                NAND_NCE | NAND_CTRL_CHANGE);
706                 return;
707
708         case NAND_CMD_READ0:
709                 chip->cmd_ctrl(mtd, NAND_CMD_READSTART,
710                                NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE);
711                 chip->cmd_ctrl(mtd, NAND_CMD_NONE,
712                                NAND_NCE | NAND_CTRL_CHANGE);
713
714                 /* This applies to read commands */
715         default:
716                 /*
717                  * If we don't have access to the busy pin, we apply the given
718                  * command delay.
719                  */
720                 if (!chip->dev_ready) {
721                         udelay(chip->chip_delay);
722                         return;
723                 }
724         }
725
726         /*
727          * Apply this short delay always to ensure that we do wait tWB in
728          * any case on any machine.
729          */
730         ndelay(100);
731
732         nand_wait_ready(mtd);
733 }
734
735 /**
736  * panic_nand_get_device - [GENERIC] Get chip for selected access
737  * @chip: the nand chip descriptor
738  * @mtd: MTD device structure
739  * @new_state: the state which is requested
740  *
741  * Used when in panic, no locks are taken.
742  */
743 static void panic_nand_get_device(struct nand_chip *chip,
744                       struct mtd_info *mtd, int new_state)
745 {
746         /* Hardware controller shared among independent devices */
747         chip->controller->active = chip;
748         chip->state = new_state;
749 }
750
751 /**
752  * nand_get_device - [GENERIC] Get chip for selected access
753  * @chip: the nand chip descriptor
754  * @mtd: MTD device structure
755  * @new_state: the state which is requested
756  *
757  * Get the device and lock it for exclusive access
758  */
759 static int
760 nand_get_device(struct nand_chip *chip, struct mtd_info *mtd, int new_state)
761 {
762         spinlock_t *lock = &chip->controller->lock;
763         wait_queue_head_t *wq = &chip->controller->wq;
764         DECLARE_WAITQUEUE(wait, current);
765 retry:
766         spin_lock(lock);
767
768         /* Hardware controller shared among independent devices */
769         if (!chip->controller->active)
770                 chip->controller->active = chip;
771
772         if (chip->controller->active == chip && chip->state == FL_READY) {
773                 chip->state = new_state;
774                 spin_unlock(lock);
775                 return 0;
776         }
777         if (new_state == FL_PM_SUSPENDED) {
778                 if (chip->controller->active->state == FL_PM_SUSPENDED) {
779                         chip->state = FL_PM_SUSPENDED;
780                         spin_unlock(lock);
781                         return 0;
782                 }
783         }
784         set_current_state(TASK_UNINTERRUPTIBLE);
785         add_wait_queue(wq, &wait);
786         spin_unlock(lock);
787         schedule();
788         remove_wait_queue(wq, &wait);
789         goto retry;
790 }
791
792 /**
793  * panic_nand_wait - [GENERIC] wait until the command is done
794  * @mtd: MTD device structure
795  * @chip: NAND chip structure
796  * @timeo: timeout
797  *
798  * Wait for command done. This is a helper function for nand_wait used when
799  * we are in interrupt context. May happen when in panic and trying to write
800  * an oops through mtdoops.
801  */
802 static void panic_nand_wait(struct mtd_info *mtd, struct nand_chip *chip,
803                             unsigned long timeo)
804 {
805         int i;
806         for (i = 0; i < timeo; i++) {
807                 if (chip->dev_ready) {
808                         if (chip->dev_ready(mtd))
809                                 break;
810                 } else {
811                         if (chip->read_byte(mtd) & NAND_STATUS_READY)
812                                 break;
813                 }
814                 mdelay(1);
815         }
816 }
817
818 /**
819  * nand_wait - [DEFAULT] wait until the command is done
820  * @mtd: MTD device structure
821  * @chip: NAND chip structure
822  *
823  * Wait for command done. This applies to erase and program only. Erase can
824  * take up to 400ms and program up to 20ms according to general NAND and
825  * SmartMedia specs.
826  */
827 static int nand_wait(struct mtd_info *mtd, struct nand_chip *chip)
828 {
829
830         unsigned long timeo = jiffies;
831         int status, state = chip->state;
832
833         if (state == FL_ERASING)
834                 timeo += (HZ * 400) / 1000;
835         else
836                 timeo += (HZ * 20) / 1000;
837
838         led_trigger_event(nand_led_trigger, LED_FULL);
839
840         /*
841          * Apply this short delay always to ensure that we do wait tWB in any
842          * case on any machine.
843          */
844         ndelay(100);
845
846         if ((state == FL_ERASING) && (chip->options & NAND_IS_AND))
847                 chip->cmdfunc(mtd, NAND_CMD_STATUS_MULTI, -1, -1);
848         else
849                 chip->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1);
850
851         if (in_interrupt() || oops_in_progress)
852                 panic_nand_wait(mtd, chip, timeo);
853         else {
854                 while (time_before(jiffies, timeo)) {
855                         if (chip->dev_ready) {
856                                 if (chip->dev_ready(mtd))
857                                         break;
858                         } else {
859                                 if (chip->read_byte(mtd) & NAND_STATUS_READY)
860                                         break;
861                         }
862                         cond_resched();
863                 }
864         }
865         led_trigger_event(nand_led_trigger, LED_OFF);
866
867         status = (int)chip->read_byte(mtd);
868         /* This can happen if in case of timeout or buggy dev_ready */
869         WARN_ON(!(status & NAND_STATUS_READY));
870         return status;
871 }
872
873 /**
874  * __nand_unlock - [REPLACEABLE] unlocks specified locked blocks
875  * @mtd: mtd info
876  * @ofs: offset to start unlock from
877  * @len: length to unlock
878  * @invert: when = 0, unlock the range of blocks within the lower and
879  *                    upper boundary address
880  *          when = 1, unlock the range of blocks outside the boundaries
881  *                    of the lower and upper boundary address
882  *
883  * Returs unlock status.
884  */
885 static int __nand_unlock(struct mtd_info *mtd, loff_t ofs,
886                                         uint64_t len, int invert)
887 {
888         int ret = 0;
889         int status, page;
890         struct nand_chip *chip = mtd->priv;
891
892         /* Submit address of first page to unlock */
893         page = ofs >> chip->page_shift;
894         chip->cmdfunc(mtd, NAND_CMD_UNLOCK1, -1, page & chip->pagemask);
895
896         /* Submit address of last page to unlock */
897         page = (ofs + len) >> chip->page_shift;
898         chip->cmdfunc(mtd, NAND_CMD_UNLOCK2, -1,
899                                 (page | invert) & chip->pagemask);
900
901         /* Call wait ready function */
902         status = chip->waitfunc(mtd, chip);
903         /* See if device thinks it succeeded */
904         if (status & NAND_STATUS_FAIL) {
905                 pr_debug("%s: error status = 0x%08x\n",
906                                         __func__, status);
907                 ret = -EIO;
908         }
909
910         return ret;
911 }
912
913 /**
914  * nand_unlock - [REPLACEABLE] unlocks specified locked blocks
915  * @mtd: mtd info
916  * @ofs: offset to start unlock from
917  * @len: length to unlock
918  *
919  * Returns unlock status.
920  */
921 int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
922 {
923         int ret = 0;
924         int chipnr;
925         struct nand_chip *chip = mtd->priv;
926
927         pr_debug("%s: start = 0x%012llx, len = %llu\n",
928                         __func__, (unsigned long long)ofs, len);
929
930         if (check_offs_len(mtd, ofs, len))
931                 ret = -EINVAL;
932
933         /* Align to last block address if size addresses end of the device */
934         if (ofs + len == mtd->size)
935                 len -= mtd->erasesize;
936
937         nand_get_device(chip, mtd, FL_UNLOCKING);
938
939         /* Shift to get chip number */
940         chipnr = ofs >> chip->chip_shift;
941
942         chip->select_chip(mtd, chipnr);
943
944         /* Check, if it is write protected */
945         if (nand_check_wp(mtd)) {
946                 pr_debug("%s: device is write protected!\n",
947                                         __func__);
948                 ret = -EIO;
949                 goto out;
950         }
951
952         ret = __nand_unlock(mtd, ofs, len, 0);
953
954 out:
955         nand_release_device(mtd);
956
957         return ret;
958 }
959 EXPORT_SYMBOL(nand_unlock);
960
961 /**
962  * nand_lock - [REPLACEABLE] locks all blocks present in the device
963  * @mtd: mtd info
964  * @ofs: offset to start unlock from
965  * @len: length to unlock
966  *
967  * This feature is not supported in many NAND parts. 'Micron' NAND parts do
968  * have this feature, but it allows only to lock all blocks, not for specified
969  * range for block. Implementing 'lock' feature by making use of 'unlock', for
970  * now.
971  *
972  * Returns lock status.
973  */
974 int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
975 {
976         int ret = 0;
977         int chipnr, status, page;
978         struct nand_chip *chip = mtd->priv;
979
980         pr_debug("%s: start = 0x%012llx, len = %llu\n",
981                         __func__, (unsigned long long)ofs, len);
982
983         if (check_offs_len(mtd, ofs, len))
984                 ret = -EINVAL;
985
986         nand_get_device(chip, mtd, FL_LOCKING);
987
988         /* Shift to get chip number */
989         chipnr = ofs >> chip->chip_shift;
990
991         chip->select_chip(mtd, chipnr);
992
993         /* Check, if it is write protected */
994         if (nand_check_wp(mtd)) {
995                 pr_debug("%s: device is write protected!\n",
996                                         __func__);
997                 status = MTD_ERASE_FAILED;
998                 ret = -EIO;
999                 goto out;
1000         }
1001
1002         /* Submit address of first page to lock */
1003         page = ofs >> chip->page_shift;
1004         chip->cmdfunc(mtd, NAND_CMD_LOCK, -1, page & chip->pagemask);
1005
1006         /* Call wait ready function */
1007         status = chip->waitfunc(mtd, chip);
1008         /* See if device thinks it succeeded */
1009         if (status & NAND_STATUS_FAIL) {
1010                 pr_debug("%s: error status = 0x%08x\n",
1011                                         __func__, status);
1012                 ret = -EIO;
1013                 goto out;
1014         }
1015
1016         ret = __nand_unlock(mtd, ofs, len, 0x1);
1017
1018 out:
1019         nand_release_device(mtd);
1020
1021         return ret;
1022 }
1023 EXPORT_SYMBOL(nand_lock);
1024
1025 /**
1026  * nand_read_page_raw - [INTERN] read raw page data without ecc
1027  * @mtd: mtd info structure
1028  * @chip: nand chip info structure
1029  * @buf: buffer to store read data
1030  * @oob_required: caller requires OOB data read to chip->oob_poi
1031  * @page: page number to read
1032  *
1033  * Not for syndrome calculating ECC controllers, which use a special oob layout.
1034  */
1035 static int nand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
1036                               uint8_t *buf, int oob_required, int page)
1037 {
1038         chip->read_buf(mtd, buf, mtd->writesize);
1039         if (oob_required)
1040                 chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
1041         return 0;
1042 }
1043
1044 /**
1045  * nand_read_page_raw_syndrome - [INTERN] read raw page data without ecc
1046  * @mtd: mtd info structure
1047  * @chip: nand chip info structure
1048  * @buf: buffer to store read data
1049  * @oob_required: caller requires OOB data read to chip->oob_poi
1050  * @page: page number to read
1051  *
1052  * We need a special oob layout and handling even when OOB isn't used.
1053  */
1054 static int nand_read_page_raw_syndrome(struct mtd_info *mtd,
1055                                        struct nand_chip *chip, uint8_t *buf,
1056                                        int oob_required, int page)
1057 {
1058         int eccsize = chip->ecc.size;
1059         int eccbytes = chip->ecc.bytes;
1060         uint8_t *oob = chip->oob_poi;
1061         int steps, size;
1062
1063         for (steps = chip->ecc.steps; steps > 0; steps--) {
1064                 chip->read_buf(mtd, buf, eccsize);
1065                 buf += eccsize;
1066
1067                 if (chip->ecc.prepad) {
1068                         chip->read_buf(mtd, oob, chip->ecc.prepad);
1069                         oob += chip->ecc.prepad;
1070                 }
1071
1072                 chip->read_buf(mtd, oob, eccbytes);
1073                 oob += eccbytes;
1074
1075                 if (chip->ecc.postpad) {
1076                         chip->read_buf(mtd, oob, chip->ecc.postpad);
1077                         oob += chip->ecc.postpad;
1078                 }
1079         }
1080
1081         size = mtd->oobsize - (oob - chip->oob_poi);
1082         if (size)
1083                 chip->read_buf(mtd, oob, size);
1084
1085         return 0;
1086 }
1087
1088 /**
1089  * nand_read_page_swecc - [REPLACEABLE] software ECC based page read function
1090  * @mtd: mtd info structure
1091  * @chip: nand chip info structure
1092  * @buf: buffer to store read data
1093  * @oob_required: caller requires OOB data read to chip->oob_poi
1094  * @page: page number to read
1095  */
1096 static int nand_read_page_swecc(struct mtd_info *mtd, struct nand_chip *chip,
1097                                 uint8_t *buf, int oob_required, int page)
1098 {
1099         int i, eccsize = chip->ecc.size;
1100         int eccbytes = chip->ecc.bytes;
1101         int eccsteps = chip->ecc.steps;
1102         uint8_t *p = buf;
1103         uint8_t *ecc_calc = chip->buffers->ecccalc;
1104         uint8_t *ecc_code = chip->buffers->ecccode;
1105         uint32_t *eccpos = chip->ecc.layout->eccpos;
1106         unsigned int max_bitflips = 0;
1107
1108         chip->ecc.read_page_raw(mtd, chip, buf, 1, page);
1109
1110         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize)
1111                 chip->ecc.calculate(mtd, p, &ecc_calc[i]);
1112
1113         for (i = 0; i < chip->ecc.total; i++)
1114                 ecc_code[i] = chip->oob_poi[eccpos[i]];
1115
1116         eccsteps = chip->ecc.steps;
1117         p = buf;
1118
1119         for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
1120                 int stat;
1121
1122                 stat = chip->ecc.correct(mtd, p, &ecc_code[i], &ecc_calc[i]);
1123                 if (stat < 0) {
1124                         mtd->ecc_stats.failed++;
1125                 } else {
1126                         mtd->ecc_stats.corrected += stat;
1127                         max_bitflips = max_t(unsigned int, max_bitflips, stat);
1128                 }
1129         }
1130         return max_bitflips;
1131 }
1132
1133 /**
1134  * nand_read_subpage - [REPLACEABLE] software ECC based sub-page read function
1135  * @mtd: mtd info structure
1136  * @chip: nand chip info structure
1137  * @data_offs: offset of requested data within the page
1138  * @readlen: data length
1139  * @bufpoi: buffer to store read data
1140  */
1141 static int nand_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
1142                         uint32_t data_offs, uint32_t readlen, uint8_t *bufpoi)
1143 {
1144         int start_step, end_step, num_steps;
1145         uint32_t *eccpos = chip->ecc.layout->eccpos;
1146         uint8_t *p;
1147         int data_col_addr, i, gaps = 0;
1148         int datafrag_len, eccfrag_len, aligned_len, aligned_pos;
1149         int busw = (chip->options & NAND_BUSWIDTH_16) ? 2 : 1;
1150         int index = 0;
1151         unsigned int max_bitflips = 0;
1152
1153         /* Column address within the page aligned to ECC size (256bytes) */
1154         start_step = data_offs / chip->ecc.size;
1155         end_step = (data_offs + readlen - 1) / chip->ecc.size;
1156         num_steps = end_step - start_step + 1;
1157
1158         /* Data size aligned to ECC ecc.size */
1159         datafrag_len = num_steps * chip->ecc.size;
1160         eccfrag_len = num_steps * chip->ecc.bytes;
1161
1162         data_col_addr = start_step * chip->ecc.size;
1163         /* If we read not a page aligned data */
1164         if (data_col_addr != 0)
1165                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, data_col_addr, -1);
1166
1167         p = bufpoi + data_col_addr;
1168         chip->read_buf(mtd, p, datafrag_len);
1169
1170         /* Calculate ECC */
1171         for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size)
1172                 chip->ecc.calculate(mtd, p, &chip->buffers->ecccalc[i]);
1173
1174         /*
1175          * The performance is faster if we position offsets according to
1176          * ecc.pos. Let's make sure that there are no gaps in ECC positions.
1177          */
1178         for (i = 0; i < eccfrag_len - 1; i++) {
1179                 if (eccpos[i + start_step * chip->ecc.bytes] + 1 !=
1180                         eccpos[i + start_step * chip->ecc.bytes + 1]) {
1181                         gaps = 1;
1182                         break;
1183                 }
1184         }
1185         if (gaps) {
1186                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, mtd->writesize, -1);
1187                 chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
1188         } else {
1189                 /*
1190                  * Send the command to read the particular ECC bytes take care
1191                  * about buswidth alignment in read_buf.
1192                  */
1193                 index = start_step * chip->ecc.bytes;
1194
1195                 aligned_pos = eccpos[index] & ~(busw - 1);
1196                 aligned_len = eccfrag_len;
1197                 if (eccpos[index] & (busw - 1))
1198                         aligned_len++;
1199                 if (eccpos[index + (num_steps * chip->ecc.bytes)] & (busw - 1))
1200                         aligned_len++;
1201
1202                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT,
1203                                         mtd->writesize + aligned_pos, -1);
1204                 chip->read_buf(mtd, &chip->oob_poi[aligned_pos], aligned_len);
1205         }
1206
1207         for (i = 0; i < eccfrag_len; i++)
1208                 chip->buffers->ecccode[i] = chip->oob_poi[eccpos[i + index]];
1209
1210         p = bufpoi + data_col_addr;
1211         for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size) {
1212                 int stat;
1213
1214                 stat = chip->ecc.correct(mtd, p,
1215                         &chip->buffers->ecccode[i], &chip->buffers->ecccalc[i]);
1216                 if (stat < 0) {
1217                         mtd->ecc_stats.failed++;
1218                 } else {
1219                         mtd->ecc_stats.corrected += stat;
1220                         max_bitflips = max_t(unsigned int, max_bitflips, stat);
1221                 }
1222         }
1223         return max_bitflips;
1224 }
1225
1226 /**
1227  * nand_read_page_hwecc - [REPLACEABLE] hardware ECC based page read function
1228  * @mtd: mtd info structure
1229  * @chip: nand chip info structure
1230  * @buf: buffer to store read data
1231  * @oob_required: caller requires OOB data read to chip->oob_poi
1232  * @page: page number to read
1233  *
1234  * Not for syndrome calculating ECC controllers which need a special oob layout.
1235  */
1236 static int nand_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip,
1237                                 uint8_t *buf, int oob_required, int page)
1238 {
1239         int i, eccsize = chip->ecc.size;
1240         int eccbytes = chip->ecc.bytes;
1241         int eccsteps = chip->ecc.steps;
1242         uint8_t *p = buf;
1243         uint8_t *ecc_calc = chip->buffers->ecccalc;
1244         uint8_t *ecc_code = chip->buffers->ecccode;
1245         uint32_t *eccpos = chip->ecc.layout->eccpos;
1246         unsigned int max_bitflips = 0;
1247
1248         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
1249                 chip->ecc.hwctl(mtd, NAND_ECC_READ);
1250                 chip->read_buf(mtd, p, eccsize);
1251                 chip->ecc.calculate(mtd, p, &ecc_calc[i]);
1252         }
1253         chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
1254
1255         for (i = 0; i < chip->ecc.total; i++)
1256                 ecc_code[i] = chip->oob_poi[eccpos[i]];
1257
1258         eccsteps = chip->ecc.steps;
1259         p = buf;
1260
1261         for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
1262                 int stat;
1263
1264                 stat = chip->ecc.correct(mtd, p, &ecc_code[i], &ecc_calc[i]);
1265                 if (stat < 0) {
1266                         mtd->ecc_stats.failed++;
1267                 } else {
1268                         mtd->ecc_stats.corrected += stat;
1269                         max_bitflips = max_t(unsigned int, max_bitflips, stat);
1270                 }
1271         }
1272         return max_bitflips;
1273 }
1274
1275 /**
1276  * nand_read_page_hwecc_oob_first - [REPLACEABLE] hw ecc, read oob first
1277  * @mtd: mtd info structure
1278  * @chip: nand chip info structure
1279  * @buf: buffer to store read data
1280  * @oob_required: caller requires OOB data read to chip->oob_poi
1281  * @page: page number to read
1282  *
1283  * Hardware ECC for large page chips, require OOB to be read first. For this
1284  * ECC mode, the write_page method is re-used from ECC_HW. These methods
1285  * read/write ECC from the OOB area, unlike the ECC_HW_SYNDROME support with
1286  * multiple ECC steps, follows the "infix ECC" scheme and reads/writes ECC from
1287  * the data area, by overwriting the NAND manufacturer bad block markings.
1288  */
1289 static int nand_read_page_hwecc_oob_first(struct mtd_info *mtd,
1290         struct nand_chip *chip, uint8_t *buf, int oob_required, int page)
1291 {
1292         int i, eccsize = chip->ecc.size;
1293         int eccbytes = chip->ecc.bytes;
1294         int eccsteps = chip->ecc.steps;
1295         uint8_t *p = buf;
1296         uint8_t *ecc_code = chip->buffers->ecccode;
1297         uint32_t *eccpos = chip->ecc.layout->eccpos;
1298         uint8_t *ecc_calc = chip->buffers->ecccalc;
1299         unsigned int max_bitflips = 0;
1300
1301         /* Read the OOB area first */
1302         chip->cmdfunc(mtd, NAND_CMD_READOOB, 0, page);
1303         chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
1304         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1305
1306         for (i = 0; i < chip->ecc.total; i++)
1307                 ecc_code[i] = chip->oob_poi[eccpos[i]];
1308
1309         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
1310                 int stat;
1311
1312                 chip->ecc.hwctl(mtd, NAND_ECC_READ);
1313                 chip->read_buf(mtd, p, eccsize);
1314                 chip->ecc.calculate(mtd, p, &ecc_calc[i]);
1315
1316                 stat = chip->ecc.correct(mtd, p, &ecc_code[i], NULL);
1317                 if (stat < 0) {
1318                         mtd->ecc_stats.failed++;
1319                 } else {
1320                         mtd->ecc_stats.corrected += stat;
1321                         max_bitflips = max_t(unsigned int, max_bitflips, stat);
1322                 }
1323         }
1324         return max_bitflips;
1325 }
1326
1327 /**
1328  * nand_read_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page read
1329  * @mtd: mtd info structure
1330  * @chip: nand chip info structure
1331  * @buf: buffer to store read data
1332  * @oob_required: caller requires OOB data read to chip->oob_poi
1333  * @page: page number to read
1334  *
1335  * The hw generator calculates the error syndrome automatically. Therefore we
1336  * need a special oob layout and handling.
1337  */
1338 static int nand_read_page_syndrome(struct mtd_info *mtd, struct nand_chip *chip,
1339                                    uint8_t *buf, int oob_required, int page)
1340 {
1341         int i, eccsize = chip->ecc.size;
1342         int eccbytes = chip->ecc.bytes;
1343         int eccsteps = chip->ecc.steps;
1344         uint8_t *p = buf;
1345         uint8_t *oob = chip->oob_poi;
1346         unsigned int max_bitflips = 0;
1347
1348         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
1349                 int stat;
1350
1351                 chip->ecc.hwctl(mtd, NAND_ECC_READ);
1352                 chip->read_buf(mtd, p, eccsize);
1353
1354                 if (chip->ecc.prepad) {
1355                         chip->read_buf(mtd, oob, chip->ecc.prepad);
1356                         oob += chip->ecc.prepad;
1357                 }
1358
1359                 chip->ecc.hwctl(mtd, NAND_ECC_READSYN);
1360                 chip->read_buf(mtd, oob, eccbytes);
1361                 stat = chip->ecc.correct(mtd, p, oob, NULL);
1362
1363                 if (stat < 0) {
1364                         mtd->ecc_stats.failed++;
1365                 } else {
1366                         mtd->ecc_stats.corrected += stat;
1367                         max_bitflips = max_t(unsigned int, max_bitflips, stat);
1368                 }
1369
1370                 oob += eccbytes;
1371
1372                 if (chip->ecc.postpad) {
1373                         chip->read_buf(mtd, oob, chip->ecc.postpad);
1374                         oob += chip->ecc.postpad;
1375                 }
1376         }
1377
1378         /* Calculate remaining oob bytes */
1379         i = mtd->oobsize - (oob - chip->oob_poi);
1380         if (i)
1381                 chip->read_buf(mtd, oob, i);
1382
1383         return max_bitflips;
1384 }
1385
1386 /**
1387  * nand_transfer_oob - [INTERN] Transfer oob to client buffer
1388  * @chip: nand chip structure
1389  * @oob: oob destination address
1390  * @ops: oob ops structure
1391  * @len: size of oob to transfer
1392  */
1393 static uint8_t *nand_transfer_oob(struct nand_chip *chip, uint8_t *oob,
1394                                   struct mtd_oob_ops *ops, size_t len)
1395 {
1396         switch (ops->mode) {
1397
1398         case MTD_OPS_PLACE_OOB:
1399         case MTD_OPS_RAW:
1400                 memcpy(oob, chip->oob_poi + ops->ooboffs, len);
1401                 return oob + len;
1402
1403         case MTD_OPS_AUTO_OOB: {
1404                 struct nand_oobfree *free = chip->ecc.layout->oobfree;
1405                 uint32_t boffs = 0, roffs = ops->ooboffs;
1406                 size_t bytes = 0;
1407
1408                 for (; free->length && len; free++, len -= bytes) {
1409                         /* Read request not from offset 0? */
1410                         if (unlikely(roffs)) {
1411                                 if (roffs >= free->length) {
1412                                         roffs -= free->length;
1413                                         continue;
1414                                 }
1415                                 boffs = free->offset + roffs;
1416                                 bytes = min_t(size_t, len,
1417                                               (free->length - roffs));
1418                                 roffs = 0;
1419                         } else {
1420                                 bytes = min_t(size_t, len, free->length);
1421                                 boffs = free->offset;
1422                         }
1423                         memcpy(oob, chip->oob_poi + boffs, bytes);
1424                         oob += bytes;
1425                 }
1426                 return oob;
1427         }
1428         default:
1429                 BUG();
1430         }
1431         return NULL;
1432 }
1433
1434 /**
1435  * nand_do_read_ops - [INTERN] Read data with ECC
1436  * @mtd: MTD device structure
1437  * @from: offset to read from
1438  * @ops: oob ops structure
1439  *
1440  * Internal function. Called with chip held.
1441  */
1442 static int nand_do_read_ops(struct mtd_info *mtd, loff_t from,
1443                             struct mtd_oob_ops *ops)
1444 {
1445         int chipnr, page, realpage, col, bytes, aligned, oob_required;
1446         struct nand_chip *chip = mtd->priv;
1447         struct mtd_ecc_stats stats;
1448         int ret = 0;
1449         uint32_t readlen = ops->len;
1450         uint32_t oobreadlen = ops->ooblen;
1451         uint32_t max_oobsize = ops->mode == MTD_OPS_AUTO_OOB ?
1452                 mtd->oobavail : mtd->oobsize;
1453
1454         uint8_t *bufpoi, *oob, *buf;
1455         unsigned int max_bitflips = 0;
1456
1457         stats = mtd->ecc_stats;
1458
1459         chipnr = (int)(from >> chip->chip_shift);
1460         chip->select_chip(mtd, chipnr);
1461
1462         realpage = (int)(from >> chip->page_shift);
1463         page = realpage & chip->pagemask;
1464
1465         col = (int)(from & (mtd->writesize - 1));
1466
1467         buf = ops->datbuf;
1468         oob = ops->oobbuf;
1469         oob_required = oob ? 1 : 0;
1470
1471         while (1) {
1472                 bytes = min(mtd->writesize - col, readlen);
1473                 aligned = (bytes == mtd->writesize);
1474
1475                 /* Is the current page in the buffer? */
1476                 if (realpage != chip->pagebuf || oob) {
1477                         bufpoi = aligned ? buf : chip->buffers->databuf;
1478
1479                         chip->cmdfunc(mtd, NAND_CMD_READ0, 0x00, page);
1480
1481                         /*
1482                          * Now read the page into the buffer.  Absent an error,
1483                          * the read methods return max bitflips per ecc step.
1484                          */
1485                         if (unlikely(ops->mode == MTD_OPS_RAW))
1486                                 ret = chip->ecc.read_page_raw(mtd, chip, bufpoi,
1487                                                               oob_required,
1488                                                               page);
1489                         else if (!aligned && NAND_HAS_SUBPAGE_READ(chip) &&
1490                                  !oob)
1491                                 ret = chip->ecc.read_subpage(mtd, chip,
1492                                                         col, bytes, bufpoi);
1493                         else
1494                                 ret = chip->ecc.read_page(mtd, chip, bufpoi,
1495                                                           oob_required, page);
1496                         if (ret < 0) {
1497                                 if (!aligned)
1498                                         /* Invalidate page cache */
1499                                         chip->pagebuf = -1;
1500                                 break;
1501                         }
1502
1503                         max_bitflips = max_t(unsigned int, max_bitflips, ret);
1504
1505                         /* Transfer not aligned data */
1506                         if (!aligned) {
1507                                 if (!NAND_HAS_SUBPAGE_READ(chip) && !oob &&
1508                                     !(mtd->ecc_stats.failed - stats.failed) &&
1509                                     (ops->mode != MTD_OPS_RAW)) {
1510                                         chip->pagebuf = realpage;
1511                                         chip->pagebuf_bitflips = ret;
1512                                 } else {
1513                                         /* Invalidate page cache */
1514                                         chip->pagebuf = -1;
1515                                 }
1516                                 memcpy(buf, chip->buffers->databuf + col, bytes);
1517                         }
1518
1519                         buf += bytes;
1520
1521                         if (unlikely(oob)) {
1522                                 int toread = min(oobreadlen, max_oobsize);
1523
1524                                 if (toread) {
1525                                         oob = nand_transfer_oob(chip,
1526                                                 oob, ops, toread);
1527                                         oobreadlen -= toread;
1528                                 }
1529                         }
1530                 } else {
1531                         memcpy(buf, chip->buffers->databuf + col, bytes);
1532                         buf += bytes;
1533                         max_bitflips = max_t(unsigned int, max_bitflips,
1534                                              chip->pagebuf_bitflips);
1535                 }
1536
1537                 readlen -= bytes;
1538
1539                 if (!readlen)
1540                         break;
1541
1542                 /* For subsequent reads align to page boundary */
1543                 col = 0;
1544                 /* Increment page address */
1545                 realpage++;
1546
1547                 page = realpage & chip->pagemask;
1548                 /* Check, if we cross a chip boundary */
1549                 if (!page) {
1550                         chipnr++;
1551                         chip->select_chip(mtd, -1);
1552                         chip->select_chip(mtd, chipnr);
1553                 }
1554         }
1555
1556         ops->retlen = ops->len - (size_t) readlen;
1557         if (oob)
1558                 ops->oobretlen = ops->ooblen - oobreadlen;
1559
1560         if (ret < 0)
1561                 return ret;
1562
1563         if (mtd->ecc_stats.failed - stats.failed)
1564                 return -EBADMSG;
1565
1566         return max_bitflips;
1567 }
1568
1569 /**
1570  * nand_read - [MTD Interface] MTD compatibility function for nand_do_read_ecc
1571  * @mtd: MTD device structure
1572  * @from: offset to read from
1573  * @len: number of bytes to read
1574  * @retlen: pointer to variable to store the number of read bytes
1575  * @buf: the databuffer to put data
1576  *
1577  * Get hold of the chip and call nand_do_read.
1578  */
1579 static int nand_read(struct mtd_info *mtd, loff_t from, size_t len,
1580                      size_t *retlen, uint8_t *buf)
1581 {
1582         struct nand_chip *chip = mtd->priv;
1583         struct mtd_oob_ops ops;
1584         int ret;
1585
1586         nand_get_device(chip, mtd, FL_READING);
1587         ops.len = len;
1588         ops.datbuf = buf;
1589         ops.oobbuf = NULL;
1590         ops.mode = MTD_OPS_PLACE_OOB;
1591         ret = nand_do_read_ops(mtd, from, &ops);
1592         *retlen = ops.retlen;
1593         nand_release_device(mtd);
1594         return ret;
1595 }
1596
1597 /**
1598  * nand_read_oob_std - [REPLACEABLE] the most common OOB data read function
1599  * @mtd: mtd info structure
1600  * @chip: nand chip info structure
1601  * @page: page number to read
1602  */
1603 static int nand_read_oob_std(struct mtd_info *mtd, struct nand_chip *chip,
1604                              int page)
1605 {
1606         chip->cmdfunc(mtd, NAND_CMD_READOOB, 0, page);
1607         chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
1608         return 0;
1609 }
1610
1611 /**
1612  * nand_read_oob_syndrome - [REPLACEABLE] OOB data read function for HW ECC
1613  *                          with syndromes
1614  * @mtd: mtd info structure
1615  * @chip: nand chip info structure
1616  * @page: page number to read
1617  */
1618 static int nand_read_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip,
1619                                   int page)
1620 {
1621         uint8_t *buf = chip->oob_poi;
1622         int length = mtd->oobsize;
1623         int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
1624         int eccsize = chip->ecc.size;
1625         uint8_t *bufpoi = buf;
1626         int i, toread, sndrnd = 0, pos;
1627
1628         chip->cmdfunc(mtd, NAND_CMD_READ0, chip->ecc.size, page);
1629         for (i = 0; i < chip->ecc.steps; i++) {
1630                 if (sndrnd) {
1631                         pos = eccsize + i * (eccsize + chunk);
1632                         if (mtd->writesize > 512)
1633                                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, pos, -1);
1634                         else
1635                                 chip->cmdfunc(mtd, NAND_CMD_READ0, pos, page);
1636                 } else
1637                         sndrnd = 1;
1638                 toread = min_t(int, length, chunk);
1639                 chip->read_buf(mtd, bufpoi, toread);
1640                 bufpoi += toread;
1641                 length -= toread;
1642         }
1643         if (length > 0)
1644                 chip->read_buf(mtd, bufpoi, length);
1645
1646         return 0;
1647 }
1648
1649 /**
1650  * nand_write_oob_std - [REPLACEABLE] the most common OOB data write function
1651  * @mtd: mtd info structure
1652  * @chip: nand chip info structure
1653  * @page: page number to write
1654  */
1655 static int nand_write_oob_std(struct mtd_info *mtd, struct nand_chip *chip,
1656                               int page)
1657 {
1658         int status = 0;
1659         const uint8_t *buf = chip->oob_poi;
1660         int length = mtd->oobsize;
1661
1662         chip->cmdfunc(mtd, NAND_CMD_SEQIN, mtd->writesize, page);
1663         chip->write_buf(mtd, buf, length);
1664         /* Send command to program the OOB data */
1665         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1666
1667         status = chip->waitfunc(mtd, chip);
1668
1669         return status & NAND_STATUS_FAIL ? -EIO : 0;
1670 }
1671
1672 /**
1673  * nand_write_oob_syndrome - [REPLACEABLE] OOB data write function for HW ECC
1674  *                           with syndrome - only for large page flash
1675  * @mtd: mtd info structure
1676  * @chip: nand chip info structure
1677  * @page: page number to write
1678  */
1679 static int nand_write_oob_syndrome(struct mtd_info *mtd,
1680                                    struct nand_chip *chip, int page)
1681 {
1682         int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
1683         int eccsize = chip->ecc.size, length = mtd->oobsize;
1684         int i, len, pos, status = 0, sndcmd = 0, steps = chip->ecc.steps;
1685         const uint8_t *bufpoi = chip->oob_poi;
1686
1687         /*
1688          * data-ecc-data-ecc ... ecc-oob
1689          * or
1690          * data-pad-ecc-pad-data-pad .... ecc-pad-oob
1691          */
1692         if (!chip->ecc.prepad && !chip->ecc.postpad) {
1693                 pos = steps * (eccsize + chunk);
1694                 steps = 0;
1695         } else
1696                 pos = eccsize;
1697
1698         chip->cmdfunc(mtd, NAND_CMD_SEQIN, pos, page);
1699         for (i = 0; i < steps; i++) {
1700                 if (sndcmd) {
1701                         if (mtd->writesize <= 512) {
1702                                 uint32_t fill = 0xFFFFFFFF;
1703
1704                                 len = eccsize;
1705                                 while (len > 0) {
1706                                         int num = min_t(int, len, 4);
1707                                         chip->write_buf(mtd, (uint8_t *)&fill,
1708                                                         num);
1709                                         len -= num;
1710                                 }
1711                         } else {
1712                                 pos = eccsize + i * (eccsize + chunk);
1713                                 chip->cmdfunc(mtd, NAND_CMD_RNDIN, pos, -1);
1714                         }
1715                 } else
1716                         sndcmd = 1;
1717                 len = min_t(int, length, chunk);
1718                 chip->write_buf(mtd, bufpoi, len);
1719                 bufpoi += len;
1720                 length -= len;
1721         }
1722         if (length > 0)
1723                 chip->write_buf(mtd, bufpoi, length);
1724
1725         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1726         status = chip->waitfunc(mtd, chip);
1727
1728         return status & NAND_STATUS_FAIL ? -EIO : 0;
1729 }
1730
1731 /**
1732  * nand_do_read_oob - [INTERN] NAND read out-of-band
1733  * @mtd: MTD device structure
1734  * @from: offset to read from
1735  * @ops: oob operations description structure
1736  *
1737  * NAND read out-of-band data from the spare area.
1738  */
1739 static int nand_do_read_oob(struct mtd_info *mtd, loff_t from,
1740                             struct mtd_oob_ops *ops)
1741 {
1742         int page, realpage, chipnr;
1743         struct nand_chip *chip = mtd->priv;
1744         struct mtd_ecc_stats stats;
1745         int readlen = ops->ooblen;
1746         int len;
1747         uint8_t *buf = ops->oobbuf;
1748         int ret = 0;
1749
1750         pr_debug("%s: from = 0x%08Lx, len = %i\n",
1751                         __func__, (unsigned long long)from, readlen);
1752
1753         stats = mtd->ecc_stats;
1754
1755         if (ops->mode == MTD_OPS_AUTO_OOB)
1756                 len = chip->ecc.layout->oobavail;
1757         else
1758                 len = mtd->oobsize;
1759
1760         if (unlikely(ops->ooboffs >= len)) {
1761                 pr_debug("%s: attempt to start read outside oob\n",
1762                                 __func__);
1763                 return -EINVAL;
1764         }
1765
1766         /* Do not allow reads past end of device */
1767         if (unlikely(from >= mtd->size ||
1768                      ops->ooboffs + readlen > ((mtd->size >> chip->page_shift) -
1769                                         (from >> chip->page_shift)) * len)) {
1770                 pr_debug("%s: attempt to read beyond end of device\n",
1771                                 __func__);
1772                 return -EINVAL;
1773         }
1774
1775         chipnr = (int)(from >> chip->chip_shift);
1776         chip->select_chip(mtd, chipnr);
1777
1778         /* Shift to get page */
1779         realpage = (int)(from >> chip->page_shift);
1780         page = realpage & chip->pagemask;
1781
1782         while (1) {
1783                 if (ops->mode == MTD_OPS_RAW)
1784                         ret = chip->ecc.read_oob_raw(mtd, chip, page);
1785                 else
1786                         ret = chip->ecc.read_oob(mtd, chip, page);
1787
1788                 if (ret < 0)
1789                         break;
1790
1791                 len = min(len, readlen);
1792                 buf = nand_transfer_oob(chip, buf, ops, len);
1793
1794                 readlen -= len;
1795                 if (!readlen)
1796                         break;
1797
1798                 /* Increment page address */
1799                 realpage++;
1800
1801                 page = realpage & chip->pagemask;
1802                 /* Check, if we cross a chip boundary */
1803                 if (!page) {
1804                         chipnr++;
1805                         chip->select_chip(mtd, -1);
1806                         chip->select_chip(mtd, chipnr);
1807                 }
1808         }
1809
1810         ops->oobretlen = ops->ooblen - readlen;
1811
1812         if (ret < 0)
1813                 return ret;
1814
1815         if (mtd->ecc_stats.failed - stats.failed)
1816                 return -EBADMSG;
1817
1818         return  mtd->ecc_stats.corrected - stats.corrected ? -EUCLEAN : 0;
1819 }
1820
1821 /**
1822  * nand_read_oob - [MTD Interface] NAND read data and/or out-of-band
1823  * @mtd: MTD device structure
1824  * @from: offset to read from
1825  * @ops: oob operation description structure
1826  *
1827  * NAND read data and/or out-of-band data.
1828  */
1829 static int nand_read_oob(struct mtd_info *mtd, loff_t from,
1830                          struct mtd_oob_ops *ops)
1831 {
1832         struct nand_chip *chip = mtd->priv;
1833         int ret = -ENOTSUPP;
1834
1835         ops->retlen = 0;
1836
1837         /* Do not allow reads past end of device */
1838         if (ops->datbuf && (from + ops->len) > mtd->size) {
1839                 pr_debug("%s: attempt to read beyond end of device\n",
1840                                 __func__);
1841                 return -EINVAL;
1842         }
1843
1844         nand_get_device(chip, mtd, FL_READING);
1845
1846         switch (ops->mode) {
1847         case MTD_OPS_PLACE_OOB:
1848         case MTD_OPS_AUTO_OOB:
1849         case MTD_OPS_RAW:
1850                 break;
1851
1852         default:
1853                 goto out;
1854         }
1855
1856         if (!ops->datbuf)
1857                 ret = nand_do_read_oob(mtd, from, ops);
1858         else
1859                 ret = nand_do_read_ops(mtd, from, ops);
1860
1861 out:
1862         nand_release_device(mtd);
1863         return ret;
1864 }
1865
1866
1867 /**
1868  * nand_write_page_raw - [INTERN] raw page write function
1869  * @mtd: mtd info structure
1870  * @chip: nand chip info structure
1871  * @buf: data buffer
1872  * @oob_required: must write chip->oob_poi to OOB
1873  *
1874  * Not for syndrome calculating ECC controllers, which use a special oob layout.
1875  */
1876 static int nand_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
1877                                 const uint8_t *buf, int oob_required)
1878 {
1879         chip->write_buf(mtd, buf, mtd->writesize);
1880         if (oob_required)
1881                 chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
1882
1883         return 0;
1884 }
1885
1886 /**
1887  * nand_write_page_raw_syndrome - [INTERN] raw page write function
1888  * @mtd: mtd info structure
1889  * @chip: nand chip info structure
1890  * @buf: data buffer
1891  * @oob_required: must write chip->oob_poi to OOB
1892  *
1893  * We need a special oob layout and handling even when ECC isn't checked.
1894  */
1895 static int nand_write_page_raw_syndrome(struct mtd_info *mtd,
1896                                         struct nand_chip *chip,
1897                                         const uint8_t *buf, int oob_required)
1898 {
1899         int eccsize = chip->ecc.size;
1900         int eccbytes = chip->ecc.bytes;
1901         uint8_t *oob = chip->oob_poi;
1902         int steps, size;
1903
1904         for (steps = chip->ecc.steps; steps > 0; steps--) {
1905                 chip->write_buf(mtd, buf, eccsize);
1906                 buf += eccsize;
1907
1908                 if (chip->ecc.prepad) {
1909                         chip->write_buf(mtd, oob, chip->ecc.prepad);
1910                         oob += chip->ecc.prepad;
1911                 }
1912
1913                 chip->read_buf(mtd, oob, eccbytes);
1914                 oob += eccbytes;
1915
1916                 if (chip->ecc.postpad) {
1917                         chip->write_buf(mtd, oob, chip->ecc.postpad);
1918                         oob += chip->ecc.postpad;
1919                 }
1920         }
1921
1922         size = mtd->oobsize - (oob - chip->oob_poi);
1923         if (size)
1924                 chip->write_buf(mtd, oob, size);
1925
1926         return 0;
1927 }
1928 /**
1929  * nand_write_page_swecc - [REPLACEABLE] software ECC based page write function
1930  * @mtd: mtd info structure
1931  * @chip: nand chip info structure
1932  * @buf: data buffer
1933  * @oob_required: must write chip->oob_poi to OOB
1934  */
1935 static int nand_write_page_swecc(struct mtd_info *mtd, struct nand_chip *chip,
1936                                   const uint8_t *buf, int oob_required)
1937 {
1938         int i, eccsize = chip->ecc.size;
1939         int eccbytes = chip->ecc.bytes;
1940         int eccsteps = chip->ecc.steps;
1941         uint8_t *ecc_calc = chip->buffers->ecccalc;
1942         const uint8_t *p = buf;
1943         uint32_t *eccpos = chip->ecc.layout->eccpos;
1944
1945         /* Software ECC calculation */
1946         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize)
1947                 chip->ecc.calculate(mtd, p, &ecc_calc[i]);
1948
1949         for (i = 0; i < chip->ecc.total; i++)
1950                 chip->oob_poi[eccpos[i]] = ecc_calc[i];
1951
1952         return chip->ecc.write_page_raw(mtd, chip, buf, 1);
1953 }
1954
1955 /**
1956  * nand_write_page_hwecc - [REPLACEABLE] hardware ECC based page write function
1957  * @mtd: mtd info structure
1958  * @chip: nand chip info structure
1959  * @buf: data buffer
1960  * @oob_required: must write chip->oob_poi to OOB
1961  */
1962 static int nand_write_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip,
1963                                   const uint8_t *buf, int oob_required)
1964 {
1965         int i, eccsize = chip->ecc.size;
1966         int eccbytes = chip->ecc.bytes;
1967         int eccsteps = chip->ecc.steps;
1968         uint8_t *ecc_calc = chip->buffers->ecccalc;
1969         const uint8_t *p = buf;
1970         uint32_t *eccpos = chip->ecc.layout->eccpos;
1971
1972         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
1973                 chip->ecc.hwctl(mtd, NAND_ECC_WRITE);
1974                 chip->write_buf(mtd, p, eccsize);
1975                 chip->ecc.calculate(mtd, p, &ecc_calc[i]);
1976         }
1977
1978         for (i = 0; i < chip->ecc.total; i++)
1979                 chip->oob_poi[eccpos[i]] = ecc_calc[i];
1980
1981         chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
1982
1983         return 0;
1984 }
1985
1986 /**
1987  * nand_write_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page write
1988  * @mtd: mtd info structure
1989  * @chip: nand chip info structure
1990  * @buf: data buffer
1991  * @oob_required: must write chip->oob_poi to OOB
1992  *
1993  * The hw generator calculates the error syndrome automatically. Therefore we
1994  * need a special oob layout and handling.
1995  */
1996 static int nand_write_page_syndrome(struct mtd_info *mtd,
1997                                     struct nand_chip *chip,
1998                                     const uint8_t *buf, int oob_required)
1999 {
2000         int i, eccsize = chip->ecc.size;
2001         int eccbytes = chip->ecc.bytes;
2002         int eccsteps = chip->ecc.steps;
2003         const uint8_t *p = buf;
2004         uint8_t *oob = chip->oob_poi;
2005
2006         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
2007
2008                 chip->ecc.hwctl(mtd, NAND_ECC_WRITE);
2009                 chip->write_buf(mtd, p, eccsize);
2010
2011                 if (chip->ecc.prepad) {
2012                         chip->write_buf(mtd, oob, chip->ecc.prepad);
2013                         oob += chip->ecc.prepad;
2014                 }
2015
2016                 chip->ecc.calculate(mtd, p, oob);
2017                 chip->write_buf(mtd, oob, eccbytes);
2018                 oob += eccbytes;
2019
2020                 if (chip->ecc.postpad) {
2021                         chip->write_buf(mtd, oob, chip->ecc.postpad);
2022                         oob += chip->ecc.postpad;
2023                 }
2024         }
2025
2026         /* Calculate remaining oob bytes */
2027         i = mtd->oobsize - (oob - chip->oob_poi);
2028         if (i)
2029                 chip->write_buf(mtd, oob, i);
2030
2031         return 0;
2032 }
2033
2034 /**
2035  * nand_write_page - [REPLACEABLE] write one page
2036  * @mtd: MTD device structure
2037  * @chip: NAND chip descriptor
2038  * @buf: the data to write
2039  * @oob_required: must write chip->oob_poi to OOB
2040  * @page: page number to write
2041  * @cached: cached programming
2042  * @raw: use _raw version of write_page
2043  */
2044 static int nand_write_page(struct mtd_info *mtd, struct nand_chip *chip,
2045                            const uint8_t *buf, int oob_required, int page,
2046                            int cached, int raw)
2047 {
2048         int status;
2049
2050         chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0x00, page);
2051
2052         if (unlikely(raw))
2053                 status = chip->ecc.write_page_raw(mtd, chip, buf, oob_required);
2054         else
2055                 status = chip->ecc.write_page(mtd, chip, buf, oob_required);
2056
2057         if (status < 0)
2058                 return status;
2059
2060         /*
2061          * Cached progamming disabled for now. Not sure if it's worth the
2062          * trouble. The speed gain is not very impressive. (2.3->2.6Mib/s).
2063          */
2064         cached = 0;
2065
2066         if (!cached || !(chip->options & NAND_CACHEPRG)) {
2067
2068                 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
2069                 status = chip->waitfunc(mtd, chip);
2070                 /*
2071                  * See if operation failed and additional status checks are
2072                  * available.
2073                  */
2074                 if ((status & NAND_STATUS_FAIL) && (chip->errstat))
2075                         status = chip->errstat(mtd, chip, FL_WRITING, status,
2076                                                page);
2077
2078                 if (status & NAND_STATUS_FAIL)
2079                         return -EIO;
2080         } else {
2081                 chip->cmdfunc(mtd, NAND_CMD_CACHEDPROG, -1, -1);
2082                 status = chip->waitfunc(mtd, chip);
2083         }
2084
2085         return 0;
2086 }
2087
2088 /**
2089  * nand_fill_oob - [INTERN] Transfer client buffer to oob
2090  * @mtd: MTD device structure
2091  * @oob: oob data buffer
2092  * @len: oob data write length
2093  * @ops: oob ops structure
2094  */
2095 static uint8_t *nand_fill_oob(struct mtd_info *mtd, uint8_t *oob, size_t len,
2096                               struct mtd_oob_ops *ops)
2097 {
2098         struct nand_chip *chip = mtd->priv;
2099
2100         /*
2101          * Initialise to all 0xFF, to avoid the possibility of left over OOB
2102          * data from a previous OOB read.
2103          */
2104         memset(chip->oob_poi, 0xff, mtd->oobsize);
2105
2106         switch (ops->mode) {
2107
2108         case MTD_OPS_PLACE_OOB:
2109         case MTD_OPS_RAW:
2110                 memcpy(chip->oob_poi + ops->ooboffs, oob, len);
2111                 return oob + len;
2112
2113         case MTD_OPS_AUTO_OOB: {
2114                 struct nand_oobfree *free = chip->ecc.layout->oobfree;
2115                 uint32_t boffs = 0, woffs = ops->ooboffs;
2116                 size_t bytes = 0;
2117
2118                 for (; free->length && len; free++, len -= bytes) {
2119                         /* Write request not from offset 0? */
2120                         if (unlikely(woffs)) {
2121                                 if (woffs >= free->length) {
2122                                         woffs -= free->length;
2123                                         continue;
2124                                 }
2125                                 boffs = free->offset + woffs;
2126                                 bytes = min_t(size_t, len,
2127                                               (free->length - woffs));
2128                                 woffs = 0;
2129                         } else {
2130                                 bytes = min_t(size_t, len, free->length);
2131                                 boffs = free->offset;
2132                         }
2133                         memcpy(chip->oob_poi + boffs, oob, bytes);
2134                         oob += bytes;
2135                 }
2136                 return oob;
2137         }
2138         default:
2139                 BUG();
2140         }
2141         return NULL;
2142 }
2143
2144 #define NOTALIGNED(x)   ((x & (chip->subpagesize - 1)) != 0)
2145
2146 /**
2147  * nand_do_write_ops - [INTERN] NAND write with ECC
2148  * @mtd: MTD device structure
2149  * @to: offset to write to
2150  * @ops: oob operations description structure
2151  *
2152  * NAND write with ECC.
2153  */
2154 static int nand_do_write_ops(struct mtd_info *mtd, loff_t to,
2155                              struct mtd_oob_ops *ops)
2156 {
2157         int chipnr, realpage, page, blockmask, column;
2158         struct nand_chip *chip = mtd->priv;
2159         uint32_t writelen = ops->len;
2160
2161         uint32_t oobwritelen = ops->ooblen;
2162         uint32_t oobmaxlen = ops->mode == MTD_OPS_AUTO_OOB ?
2163                                 mtd->oobavail : mtd->oobsize;
2164
2165         uint8_t *oob = ops->oobbuf;
2166         uint8_t *buf = ops->datbuf;
2167         int ret, subpage;
2168         int oob_required = oob ? 1 : 0;
2169
2170         ops->retlen = 0;
2171         if (!writelen)
2172                 return 0;
2173
2174         /* Reject writes, which are not page aligned */
2175         if (NOTALIGNED(to) || NOTALIGNED(ops->len)) {
2176                 pr_notice("%s: attempt to write non page aligned data\n",
2177                            __func__);
2178                 return -EINVAL;
2179         }
2180
2181         column = to & (mtd->writesize - 1);
2182         subpage = column || (writelen & (mtd->writesize - 1));
2183
2184         if (subpage && oob)
2185                 return -EINVAL;
2186
2187         chipnr = (int)(to >> chip->chip_shift);
2188         chip->select_chip(mtd, chipnr);
2189
2190         /* Check, if it is write protected */
2191         if (nand_check_wp(mtd))
2192                 return -EIO;
2193
2194         realpage = (int)(to >> chip->page_shift);
2195         page = realpage & chip->pagemask;
2196         blockmask = (1 << (chip->phys_erase_shift - chip->page_shift)) - 1;
2197
2198         /* Invalidate the page cache, when we write to the cached page */
2199         if (to <= (chip->pagebuf << chip->page_shift) &&
2200             (chip->pagebuf << chip->page_shift) < (to + ops->len))
2201                 chip->pagebuf = -1;
2202
2203         /* Don't allow multipage oob writes with offset */
2204         if (oob && ops->ooboffs && (ops->ooboffs + ops->ooblen > oobmaxlen))
2205                 return -EINVAL;
2206
2207         while (1) {
2208                 int bytes = mtd->writesize;
2209                 int cached = writelen > bytes && page != blockmask;
2210                 uint8_t *wbuf = buf;
2211
2212                 /* Partial page write? */
2213                 if (unlikely(column || writelen < (mtd->writesize - 1))) {
2214                         cached = 0;
2215                         bytes = min_t(int, bytes - column, (int) writelen);
2216                         chip->pagebuf = -1;
2217                         memset(chip->buffers->databuf, 0xff, mtd->writesize);
2218                         memcpy(&chip->buffers->databuf[column], buf, bytes);
2219                         wbuf = chip->buffers->databuf;
2220                 }
2221
2222                 if (unlikely(oob)) {
2223                         size_t len = min(oobwritelen, oobmaxlen);
2224                         oob = nand_fill_oob(mtd, oob, len, ops);
2225                         oobwritelen -= len;
2226                 } else {
2227                         /* We still need to erase leftover OOB data */
2228                         memset(chip->oob_poi, 0xff, mtd->oobsize);
2229                 }
2230
2231                 ret = chip->write_page(mtd, chip, wbuf, oob_required, page,
2232                                        cached, (ops->mode == MTD_OPS_RAW));
2233                 if (ret)
2234                         break;
2235
2236                 writelen -= bytes;
2237                 if (!writelen)
2238                         break;
2239
2240                 column = 0;
2241                 buf += bytes;
2242                 realpage++;
2243
2244                 page = realpage & chip->pagemask;
2245                 /* Check, if we cross a chip boundary */
2246                 if (!page) {
2247                         chipnr++;
2248                         chip->select_chip(mtd, -1);
2249                         chip->select_chip(mtd, chipnr);
2250                 }
2251         }
2252
2253         ops->retlen = ops->len - writelen;
2254         if (unlikely(oob))
2255                 ops->oobretlen = ops->ooblen;
2256         return ret;
2257 }
2258
2259 /**
2260  * panic_nand_write - [MTD Interface] NAND write with ECC
2261  * @mtd: MTD device structure
2262  * @to: offset to write to
2263  * @len: number of bytes to write
2264  * @retlen: pointer to variable to store the number of written bytes
2265  * @buf: the data to write
2266  *
2267  * NAND write with ECC. Used when performing writes in interrupt context, this
2268  * may for example be called by mtdoops when writing an oops while in panic.
2269  */
2270 static int panic_nand_write(struct mtd_info *mtd, loff_t to, size_t len,
2271                             size_t *retlen, const uint8_t *buf)
2272 {
2273         struct nand_chip *chip = mtd->priv;
2274         struct mtd_oob_ops ops;
2275         int ret;
2276
2277         /* Wait for the device to get ready */
2278         panic_nand_wait(mtd, chip, 400);
2279
2280         /* Grab the device */
2281         panic_nand_get_device(chip, mtd, FL_WRITING);
2282
2283         ops.len = len;
2284         ops.datbuf = (uint8_t *)buf;
2285         ops.oobbuf = NULL;
2286         ops.mode = MTD_OPS_PLACE_OOB;
2287
2288         ret = nand_do_write_ops(mtd, to, &ops);
2289
2290         *retlen = ops.retlen;
2291         return ret;
2292 }
2293
2294 /**
2295  * nand_write - [MTD Interface] NAND write with ECC
2296  * @mtd: MTD device structure
2297  * @to: offset to write to
2298  * @len: number of bytes to write
2299  * @retlen: pointer to variable to store the number of written bytes
2300  * @buf: the data to write
2301  *
2302  * NAND write with ECC.
2303  */
2304 static int nand_write(struct mtd_info *mtd, loff_t to, size_t len,
2305                           size_t *retlen, const uint8_t *buf)
2306 {
2307         struct nand_chip *chip = mtd->priv;
2308         struct mtd_oob_ops ops;
2309         int ret;
2310
2311         nand_get_device(chip, mtd, FL_WRITING);
2312         ops.len = len;
2313         ops.datbuf = (uint8_t *)buf;
2314         ops.oobbuf = NULL;
2315         ops.mode = MTD_OPS_PLACE_OOB;
2316         ret = nand_do_write_ops(mtd, to, &ops);
2317         *retlen = ops.retlen;
2318         nand_release_device(mtd);
2319         return ret;
2320 }
2321
2322 /**
2323  * nand_do_write_oob - [MTD Interface] NAND write out-of-band
2324  * @mtd: MTD device structure
2325  * @to: offset to write to
2326  * @ops: oob operation description structure
2327  *
2328  * NAND write out-of-band.
2329  */
2330 static int nand_do_write_oob(struct mtd_info *mtd, loff_t to,
2331                              struct mtd_oob_ops *ops)
2332 {
2333         int chipnr, page, status, len;
2334         struct nand_chip *chip = mtd->priv;
2335
2336         pr_debug("%s: to = 0x%08x, len = %i\n",
2337                          __func__, (unsigned int)to, (int)ops->ooblen);
2338
2339         if (ops->mode == MTD_OPS_AUTO_OOB)
2340                 len = chip->ecc.layout->oobavail;
2341         else
2342                 len = mtd->oobsize;
2343
2344         /* Do not allow write past end of page */
2345         if ((ops->ooboffs + ops->ooblen) > len) {
2346                 pr_debug("%s: attempt to write past end of page\n",
2347                                 __func__);
2348                 return -EINVAL;
2349         }
2350
2351         if (unlikely(ops->ooboffs >= len)) {
2352                 pr_debug("%s: attempt to start write outside oob\n",
2353                                 __func__);
2354                 return -EINVAL;
2355         }
2356
2357         /* Do not allow write past end of device */
2358         if (unlikely(to >= mtd->size ||
2359                      ops->ooboffs + ops->ooblen >
2360                         ((mtd->size >> chip->page_shift) -
2361                          (to >> chip->page_shift)) * len)) {
2362                 pr_debug("%s: attempt to write beyond end of device\n",
2363                                 __func__);
2364                 return -EINVAL;
2365         }
2366
2367         chipnr = (int)(to >> chip->chip_shift);
2368         chip->select_chip(mtd, chipnr);
2369
2370         /* Shift to get page */
2371         page = (int)(to >> chip->page_shift);
2372
2373         /*
2374          * Reset the chip. Some chips (like the Toshiba TC5832DC found in one
2375          * of my DiskOnChip 2000 test units) will clear the whole data page too
2376          * if we don't do this. I have no clue why, but I seem to have 'fixed'
2377          * it in the doc2000 driver in August 1999.  dwmw2.
2378          */
2379         chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
2380
2381         /* Check, if it is write protected */
2382         if (nand_check_wp(mtd))
2383                 return -EROFS;
2384
2385         /* Invalidate the page cache, if we write to the cached page */
2386         if (page == chip->pagebuf)
2387                 chip->pagebuf = -1;
2388
2389         nand_fill_oob(mtd, ops->oobbuf, ops->ooblen, ops);
2390
2391         if (ops->mode == MTD_OPS_RAW)
2392                 status = chip->ecc.write_oob_raw(mtd, chip, page & chip->pagemask);
2393         else
2394                 status = chip->ecc.write_oob(mtd, chip, page & chip->pagemask);
2395
2396         if (status)
2397                 return status;
2398
2399         ops->oobretlen = ops->ooblen;
2400
2401         return 0;
2402 }
2403
2404 /**
2405  * nand_write_oob - [MTD Interface] NAND write data and/or out-of-band
2406  * @mtd: MTD device structure
2407  * @to: offset to write to
2408  * @ops: oob operation description structure
2409  */
2410 static int nand_write_oob(struct mtd_info *mtd, loff_t to,
2411                           struct mtd_oob_ops *ops)
2412 {
2413         struct nand_chip *chip = mtd->priv;
2414         int ret = -ENOTSUPP;
2415
2416         ops->retlen = 0;
2417
2418         /* Do not allow writes past end of device */
2419         if (ops->datbuf && (to + ops->len) > mtd->size) {
2420                 pr_debug("%s: attempt to write beyond end of device\n",
2421                                 __func__);
2422                 return -EINVAL;
2423         }
2424
2425         nand_get_device(chip, mtd, FL_WRITING);
2426
2427         switch (ops->mode) {
2428         case MTD_OPS_PLACE_OOB:
2429         case MTD_OPS_AUTO_OOB:
2430         case MTD_OPS_RAW:
2431                 break;
2432
2433         default:
2434                 goto out;
2435         }
2436
2437         if (!ops->datbuf)
2438                 ret = nand_do_write_oob(mtd, to, ops);
2439         else
2440                 ret = nand_do_write_ops(mtd, to, ops);
2441
2442 out:
2443         nand_release_device(mtd);
2444         return ret;
2445 }
2446
2447 /**
2448  * single_erase_cmd - [GENERIC] NAND standard block erase command function
2449  * @mtd: MTD device structure
2450  * @page: the page address of the block which will be erased
2451  *
2452  * Standard erase command for NAND chips.
2453  */
2454 static void single_erase_cmd(struct mtd_info *mtd, int page)
2455 {
2456         struct nand_chip *chip = mtd->priv;
2457         /* Send commands to erase a block */
2458         chip->cmdfunc(mtd, NAND_CMD_ERASE1, -1, page);
2459         chip->cmdfunc(mtd, NAND_CMD_ERASE2, -1, -1);
2460 }
2461
2462 /**
2463  * multi_erase_cmd - [GENERIC] AND specific block erase command function
2464  * @mtd: MTD device structure
2465  * @page: the page address of the block which will be erased
2466  *
2467  * AND multi block erase command function. Erase 4 consecutive blocks.
2468  */
2469 static void multi_erase_cmd(struct mtd_info *mtd, int page)
2470 {
2471         struct nand_chip *chip = mtd->priv;
2472         /* Send commands to erase a block */
2473         chip->cmdfunc(mtd, NAND_CMD_ERASE1, -1, page++);
2474         chip->cmdfunc(mtd, NAND_CMD_ERASE1, -1, page++);
2475         chip->cmdfunc(mtd, NAND_CMD_ERASE1, -1, page++);
2476         chip->cmdfunc(mtd, NAND_CMD_ERASE1, -1, page);
2477         chip->cmdfunc(mtd, NAND_CMD_ERASE2, -1, -1);
2478 }
2479
2480 /**
2481  * nand_erase - [MTD Interface] erase block(s)
2482  * @mtd: MTD device structure
2483  * @instr: erase instruction
2484  *
2485  * Erase one ore more blocks.
2486  */
2487 static int nand_erase(struct mtd_info *mtd, struct erase_info *instr)
2488 {
2489         return nand_erase_nand(mtd, instr, 0);
2490 }
2491
2492 #define BBT_PAGE_MASK   0xffffff3f
2493 /**
2494  * nand_erase_nand - [INTERN] erase block(s)
2495  * @mtd: MTD device structure
2496  * @instr: erase instruction
2497  * @allowbbt: allow erasing the bbt area
2498  *
2499  * Erase one ore more blocks.
2500  */
2501 int nand_erase_nand(struct mtd_info *mtd, struct erase_info *instr,
2502                     int allowbbt)
2503 {
2504         int page, status, pages_per_block, ret, chipnr;
2505         struct nand_chip *chip = mtd->priv;
2506         loff_t rewrite_bbt[NAND_MAX_CHIPS] = {0};
2507         unsigned int bbt_masked_page = 0xffffffff;
2508         loff_t len;
2509
2510         pr_debug("%s: start = 0x%012llx, len = %llu\n",
2511                         __func__, (unsigned long long)instr->addr,
2512                         (unsigned long long)instr->len);
2513
2514         if (check_offs_len(mtd, instr->addr, instr->len))
2515                 return -EINVAL;
2516
2517         /* Grab the lock and see if the device is available */
2518         nand_get_device(chip, mtd, FL_ERASING);
2519
2520         /* Shift to get first page */
2521         page = (int)(instr->addr >> chip->page_shift);
2522         chipnr = (int)(instr->addr >> chip->chip_shift);
2523
2524         /* Calculate pages in each block */
2525         pages_per_block = 1 << (chip->phys_erase_shift - chip->page_shift);
2526
2527         /* Select the NAND device */
2528         chip->select_chip(mtd, chipnr);
2529
2530         /* Check, if it is write protected */
2531         if (nand_check_wp(mtd)) {
2532                 pr_debug("%s: device is write protected!\n",
2533                                 __func__);
2534                 instr->state = MTD_ERASE_FAILED;
2535                 goto erase_exit;
2536         }
2537
2538         /*
2539          * If BBT requires refresh, set the BBT page mask to see if the BBT
2540          * should be rewritten. Otherwise the mask is set to 0xffffffff which
2541          * can not be matched. This is also done when the bbt is actually
2542          * erased to avoid recursive updates.
2543          */
2544         if (chip->options & BBT_AUTO_REFRESH && !allowbbt)
2545                 bbt_masked_page = chip->bbt_td->pages[chipnr] & BBT_PAGE_MASK;
2546
2547         /* Loop through the pages */
2548         len = instr->len;
2549
2550         instr->state = MTD_ERASING;
2551
2552         while (len) {
2553                 /* Check if we have a bad block, we do not erase bad blocks! */
2554                 if (nand_block_checkbad(mtd, ((loff_t) page) <<
2555                                         chip->page_shift, 0, allowbbt)) {
2556                         pr_warn("%s: attempt to erase a bad block at page 0x%08x\n",
2557                                     __func__, page);
2558                         instr->state = MTD_ERASE_FAILED;
2559                         goto erase_exit;
2560                 }
2561
2562                 /*
2563                  * Invalidate the page cache, if we erase the block which
2564                  * contains the current cached page.
2565                  */
2566                 if (page <= chip->pagebuf && chip->pagebuf <
2567                     (page + pages_per_block))
2568                         chip->pagebuf = -1;
2569
2570                 chip->erase_cmd(mtd, page & chip->pagemask);
2571
2572                 status = chip->waitfunc(mtd, chip);
2573
2574                 /*
2575                  * See if operation failed and additional status checks are
2576                  * available
2577                  */
2578                 if ((status & NAND_STATUS_FAIL) && (chip->errstat))
2579                         status = chip->errstat(mtd, chip, FL_ERASING,
2580                                                status, page);
2581
2582                 /* See if block erase succeeded */
2583                 if (status & NAND_STATUS_FAIL) {
2584                         pr_debug("%s: failed erase, page 0x%08x\n",
2585                                         __func__, page);
2586                         instr->state = MTD_ERASE_FAILED;
2587                         instr->fail_addr =
2588                                 ((loff_t)page << chip->page_shift);
2589                         goto erase_exit;
2590                 }
2591
2592                 /*
2593                  * If BBT requires refresh, set the BBT rewrite flag to the
2594                  * page being erased.
2595                  */
2596                 if (bbt_masked_page != 0xffffffff &&
2597                     (page & BBT_PAGE_MASK) == bbt_masked_page)
2598                             rewrite_bbt[chipnr] =
2599                                         ((loff_t)page << chip->page_shift);
2600
2601                 /* Increment page address and decrement length */
2602                 len -= (1 << chip->phys_erase_shift);
2603                 page += pages_per_block;
2604
2605                 /* Check, if we cross a chip boundary */
2606                 if (len && !(page & chip->pagemask)) {
2607                         chipnr++;
2608                         chip->select_chip(mtd, -1);
2609                         chip->select_chip(mtd, chipnr);
2610
2611                         /*
2612                          * If BBT requires refresh and BBT-PERCHIP, set the BBT
2613                          * page mask to see if this BBT should be rewritten.
2614                          */
2615                         if (bbt_masked_page != 0xffffffff &&
2616                             (chip->bbt_td->options & NAND_BBT_PERCHIP))
2617                                 bbt_masked_page = chip->bbt_td->pages[chipnr] &
2618                                         BBT_PAGE_MASK;
2619                 }
2620         }
2621         instr->state = MTD_ERASE_DONE;
2622
2623 erase_exit:
2624
2625         ret = instr->state == MTD_ERASE_DONE ? 0 : -EIO;
2626
2627         /* Deselect and wake up anyone waiting on the device */
2628         nand_release_device(mtd);
2629
2630         /* Do call back function */
2631         if (!ret)
2632                 mtd_erase_callback(instr);
2633
2634         /*
2635          * If BBT requires refresh and erase was successful, rewrite any
2636          * selected bad block tables.
2637          */
2638         if (bbt_masked_page == 0xffffffff || ret)
2639                 return ret;
2640
2641         for (chipnr = 0; chipnr < chip->numchips; chipnr++) {
2642                 if (!rewrite_bbt[chipnr])
2643                         continue;
2644                 /* Update the BBT for chip */
2645                 pr_debug("%s: nand_update_bbt (%d:0x%0llx 0x%0x)\n",
2646                                 __func__, chipnr, rewrite_bbt[chipnr],
2647                                 chip->bbt_td->pages[chipnr]);
2648                 nand_update_bbt(mtd, rewrite_bbt[chipnr]);
2649         }
2650
2651         /* Return more or less happy */
2652         return ret;
2653 }
2654
2655 /**
2656  * nand_sync - [MTD Interface] sync
2657  * @mtd: MTD device structure
2658  *
2659  * Sync is actually a wait for chip ready function.
2660  */
2661 static void nand_sync(struct mtd_info *mtd)
2662 {
2663         struct nand_chip *chip = mtd->priv;
2664
2665         pr_debug("%s: called\n", __func__);
2666
2667         /* Grab the lock and see if the device is available */
2668         nand_get_device(chip, mtd, FL_SYNCING);
2669         /* Release it and go back */
2670         nand_release_device(mtd);
2671 }
2672
2673 /**
2674  * nand_block_isbad - [MTD Interface] Check if block at offset is bad
2675  * @mtd: MTD device structure
2676  * @offs: offset relative to mtd start
2677  */
2678 static int nand_block_isbad(struct mtd_info *mtd, loff_t offs)
2679 {
2680         return nand_block_checkbad(mtd, offs, 1, 0);
2681 }
2682
2683 /**
2684  * nand_block_markbad - [MTD Interface] Mark block at the given offset as bad
2685  * @mtd: MTD device structure
2686  * @ofs: offset relative to mtd start
2687  */
2688 static int nand_block_markbad(struct mtd_info *mtd, loff_t ofs)
2689 {
2690         struct nand_chip *chip = mtd->priv;
2691         int ret;
2692
2693         ret = nand_block_isbad(mtd, ofs);
2694         if (ret) {
2695                 /* If it was bad already, return success and do nothing */
2696                 if (ret > 0)
2697                         return 0;
2698                 return ret;
2699         }
2700
2701         return chip->block_markbad(mtd, ofs);
2702 }
2703
2704 /**
2705  * nand_onfi_set_features- [REPLACEABLE] set features for ONFI nand
2706  * @mtd: MTD device structure
2707  * @chip: nand chip info structure
2708  * @addr: feature address.
2709  * @subfeature_param: the subfeature parameters, a four bytes array.
2710  */
2711 static int nand_onfi_set_features(struct mtd_info *mtd, struct nand_chip *chip,
2712                         int addr, uint8_t *subfeature_param)
2713 {
2714         int status;
2715
2716         if (!chip->onfi_version)
2717                 return -EINVAL;
2718
2719         chip->cmdfunc(mtd, NAND_CMD_SET_FEATURES, addr, -1);
2720         chip->write_buf(mtd, subfeature_param, ONFI_SUBFEATURE_PARAM_LEN);
2721         status = chip->waitfunc(mtd, chip);
2722         if (status & NAND_STATUS_FAIL)
2723                 return -EIO;
2724         return 0;
2725 }
2726
2727 /**
2728  * nand_onfi_get_features- [REPLACEABLE] get features for ONFI nand
2729  * @mtd: MTD device structure
2730  * @chip: nand chip info structure
2731  * @addr: feature address.
2732  * @subfeature_param: the subfeature parameters, a four bytes array.
2733  */
2734 static int nand_onfi_get_features(struct mtd_info *mtd, struct nand_chip *chip,
2735                         int addr, uint8_t *subfeature_param)
2736 {
2737         if (!chip->onfi_version)
2738                 return -EINVAL;
2739
2740         /* clear the sub feature parameters */
2741         memset(subfeature_param, 0, ONFI_SUBFEATURE_PARAM_LEN);
2742
2743         chip->cmdfunc(mtd, NAND_CMD_GET_FEATURES, addr, -1);
2744         chip->read_buf(mtd, subfeature_param, ONFI_SUBFEATURE_PARAM_LEN);
2745         return 0;
2746 }
2747
2748 /**
2749  * nand_suspend - [MTD Interface] Suspend the NAND flash
2750  * @mtd: MTD device structure
2751  */
2752 static int nand_suspend(struct mtd_info *mtd)
2753 {
2754         struct nand_chip *chip = mtd->priv;
2755
2756         return nand_get_device(chip, mtd, FL_PM_SUSPENDED);
2757 }
2758
2759 /**
2760  * nand_resume - [MTD Interface] Resume the NAND flash
2761  * @mtd: MTD device structure
2762  */
2763 static void nand_resume(struct mtd_info *mtd)
2764 {
2765         struct nand_chip *chip = mtd->priv;
2766
2767         if (chip->state == FL_PM_SUSPENDED)
2768                 nand_release_device(mtd);
2769         else
2770                 pr_err("%s called for a chip which is not in suspended state\n",
2771                         __func__);
2772 }
2773
2774 /* Set default functions */
2775 static void nand_set_defaults(struct nand_chip *chip, int busw)
2776 {
2777         /* check for proper chip_delay setup, set 20us if not */
2778         if (!chip->chip_delay)
2779                 chip->chip_delay = 20;
2780
2781         /* check, if a user supplied command function given */
2782         if (chip->cmdfunc == NULL)
2783                 chip->cmdfunc = nand_command;
2784
2785         /* check, if a user supplied wait function given */
2786         if (chip->waitfunc == NULL)
2787                 chip->waitfunc = nand_wait;
2788
2789         if (!chip->select_chip)
2790                 chip->select_chip = nand_select_chip;
2791         if (!chip->read_byte)
2792                 chip->read_byte = busw ? nand_read_byte16 : nand_read_byte;
2793         if (!chip->read_word)
2794                 chip->read_word = nand_read_word;
2795         if (!chip->block_bad)
2796                 chip->block_bad = nand_block_bad;
2797         if (!chip->block_markbad)
2798                 chip->block_markbad = nand_default_block_markbad;
2799         if (!chip->write_buf)
2800                 chip->write_buf = busw ? nand_write_buf16 : nand_write_buf;
2801         if (!chip->read_buf)
2802                 chip->read_buf = busw ? nand_read_buf16 : nand_read_buf;
2803         if (!chip->scan_bbt)
2804                 chip->scan_bbt = nand_default_bbt;
2805
2806         if (!chip->controller) {
2807                 chip->controller = &chip->hwcontrol;
2808                 spin_lock_init(&chip->controller->lock);
2809                 init_waitqueue_head(&chip->controller->wq);
2810         }
2811
2812 }
2813
2814 /* Sanitize ONFI strings so we can safely print them */
2815 static void sanitize_string(uint8_t *s, size_t len)
2816 {
2817         ssize_t i;
2818
2819         /* Null terminate */
2820         s[len - 1] = 0;
2821
2822         /* Remove non printable chars */
2823         for (i = 0; i < len - 1; i++) {
2824                 if (s[i] < ' ' || s[i] > 127)
2825                         s[i] = '?';
2826         }
2827
2828         /* Remove trailing spaces */
2829         strim(s);
2830 }
2831
2832 static u16 onfi_crc16(u16 crc, u8 const *p, size_t len)
2833 {
2834         int i;
2835         while (len--) {
2836                 crc ^= *p++ << 8;
2837                 for (i = 0; i < 8; i++)
2838                         crc = (crc << 1) ^ ((crc & 0x8000) ? 0x8005 : 0);
2839         }
2840
2841         return crc;
2842 }
2843
2844 /*
2845  * Check if the NAND chip is ONFI compliant, returns 1 if it is, 0 otherwise.
2846  */
2847 static int nand_flash_detect_onfi(struct mtd_info *mtd, struct nand_chip *chip,
2848                                         int *busw)
2849 {
2850         struct nand_onfi_params *p = &chip->onfi_params;
2851         int i;
2852         int val;
2853
2854         /* Try ONFI for unknown chip or LP */
2855         chip->cmdfunc(mtd, NAND_CMD_READID, 0x20, -1);
2856         if (chip->read_byte(mtd) != 'O' || chip->read_byte(mtd) != 'N' ||
2857                 chip->read_byte(mtd) != 'F' || chip->read_byte(mtd) != 'I')
2858                 return 0;
2859
2860         chip->cmdfunc(mtd, NAND_CMD_PARAM, 0, -1);
2861         for (i = 0; i < 3; i++) {
2862                 chip->read_buf(mtd, (uint8_t *)p, sizeof(*p));
2863                 if (onfi_crc16(ONFI_CRC_BASE, (uint8_t *)p, 254) ==
2864                                 le16_to_cpu(p->crc)) {
2865                         pr_info("ONFI param page %d valid\n", i);
2866                         break;
2867                 }
2868         }
2869
2870         if (i == 3)
2871                 return 0;
2872
2873         /* Check version */
2874         val = le16_to_cpu(p->revision);
2875         if (val & (1 << 5))
2876                 chip->onfi_version = 23;
2877         else if (val & (1 << 4))
2878                 chip->onfi_version = 22;
2879         else if (val & (1 << 3))
2880                 chip->onfi_version = 21;
2881         else if (val & (1 << 2))
2882                 chip->onfi_version = 20;
2883         else if (val & (1 << 1))
2884                 chip->onfi_version = 10;
2885         else
2886                 chip->onfi_version = 0;
2887
2888         if (!chip->onfi_version) {
2889                 pr_info("%s: unsupported ONFI version: %d\n", __func__, val);
2890                 return 0;
2891         }
2892
2893         sanitize_string(p->manufacturer, sizeof(p->manufacturer));
2894         sanitize_string(p->model, sizeof(p->model));
2895         if (!mtd->name)
2896                 mtd->name = p->model;
2897         mtd->writesize = le32_to_cpu(p->byte_per_page);
2898         mtd->erasesize = le32_to_cpu(p->pages_per_block) * mtd->writesize;
2899         mtd->oobsize = le16_to_cpu(p->spare_bytes_per_page);
2900         chip->chipsize = le32_to_cpu(p->blocks_per_lun);
2901         chip->chipsize *= (uint64_t)mtd->erasesize * p->lun_count;
2902         *busw = 0;
2903         if (le16_to_cpu(p->features) & 1)
2904                 *busw = NAND_BUSWIDTH_16;
2905
2906         pr_info("ONFI flash detected\n");
2907         return 1;
2908 }
2909
2910 /*
2911  * nand_id_has_period - Check if an ID string has a given wraparound period
2912  * @id_data: the ID string
2913  * @arrlen: the length of the @id_data array
2914  * @period: the period of repitition
2915  *
2916  * Check if an ID string is repeated within a given sequence of bytes at
2917  * specific repetition interval period (e.g., {0x20,0x01,0x7F,0x20} has a
2918  * period of 2). This is a helper function for nand_id_len(). Returns non-zero
2919  * if the repetition has a period of @period; otherwise, returns zero.
2920  */
2921 static int nand_id_has_period(u8 *id_data, int arrlen, int period)
2922 {
2923         int i, j;
2924         for (i = 0; i < period; i++)
2925                 for (j = i + period; j < arrlen; j += period)
2926                         if (id_data[i] != id_data[j])
2927                                 return 0;
2928         return 1;
2929 }
2930
2931 /*
2932  * nand_id_len - Get the length of an ID string returned by CMD_READID
2933  * @id_data: the ID string
2934  * @arrlen: the length of the @id_data array
2935
2936  * Returns the length of the ID string, according to known wraparound/trailing
2937  * zero patterns. If no pattern exists, returns the length of the array.
2938  */
2939 static int nand_id_len(u8 *id_data, int arrlen)
2940 {
2941         int last_nonzero, period;
2942
2943         /* Find last non-zero byte */
2944         for (last_nonzero = arrlen - 1; last_nonzero >= 0; last_nonzero--)
2945                 if (id_data[last_nonzero])
2946                         break;
2947
2948         /* All zeros */
2949         if (last_nonzero < 0)
2950                 return 0;
2951
2952         /* Calculate wraparound period */
2953         for (period = 1; period < arrlen; period++)
2954                 if (nand_id_has_period(id_data, arrlen, period))
2955                         break;
2956
2957         /* There's a repeated pattern */
2958         if (period < arrlen)
2959                 return period;
2960
2961         /* There are trailing zeros */
2962         if (last_nonzero < arrlen - 1)
2963                 return last_nonzero + 1;
2964
2965         /* No pattern detected */
2966         return arrlen;
2967 }
2968
2969 /*
2970  * Many new NAND share similar device ID codes, which represent the size of the
2971  * chip. The rest of the parameters must be decoded according to generic or
2972  * manufacturer-specific "extended ID" decoding patterns.
2973  */
2974 static void nand_decode_ext_id(struct mtd_info *mtd, struct nand_chip *chip,
2975                                 u8 id_data[8], int *busw)
2976 {
2977         int extid, id_len;
2978         /* The 3rd id byte holds MLC / multichip data */
2979         chip->cellinfo = id_data[2];
2980         /* The 4th id byte is the important one */
2981         extid = id_data[3];
2982
2983         id_len = nand_id_len(id_data, 8);
2984
2985         /*
2986          * Field definitions are in the following datasheets:
2987          * Old style (4,5 byte ID): Samsung K9GAG08U0M (p.32)
2988          * New Samsung (6 byte ID): Samsung K9GAG08U0F (p.44)
2989          * Hynix MLC   (6 byte ID): Hynix H27UBG8T2B (p.22)
2990          *
2991          * Check for ID length, non-zero 6th byte, cell type, and Hynix/Samsung
2992          * ID to decide what to do.
2993          */
2994         if (id_len == 6 && id_data[0] == NAND_MFR_SAMSUNG &&
2995                         (chip->cellinfo & NAND_CI_CELLTYPE_MSK) &&
2996                         id_data[5] != 0x00) {
2997                 /* Calc pagesize */
2998                 mtd->writesize = 2048 << (extid & 0x03);
2999                 extid >>= 2;
3000                 /* Calc oobsize */
3001                 switch (((extid >> 2) & 0x04) | (extid & 0x03)) {
3002                 case 1:
3003                         mtd->oobsize = 128;
3004                         break;
3005                 case 2:
3006                         mtd->oobsize = 218;
3007                         break;
3008                 case 3:
3009                         mtd->oobsize = 400;
3010                         break;
3011                 case 4:
3012                         mtd->oobsize = 436;
3013                         break;
3014                 case 5:
3015                         mtd->oobsize = 512;
3016                         break;
3017                 case 6:
3018                 default: /* Other cases are "reserved" (unknown) */
3019                         mtd->oobsize = 640;
3020                         break;
3021                 }
3022                 extid >>= 2;
3023                 /* Calc blocksize */
3024                 mtd->erasesize = (128 * 1024) <<
3025                         (((extid >> 1) & 0x04) | (extid & 0x03));
3026                 *busw = 0;
3027         } else if (id_len == 6 && id_data[0] == NAND_MFR_HYNIX &&
3028                         (chip->cellinfo & NAND_CI_CELLTYPE_MSK)) {
3029                 unsigned int tmp;
3030
3031                 /* Calc pagesize */
3032                 mtd->writesize = 2048 << (extid & 0x03);
3033                 extid >>= 2;
3034                 /* Calc oobsize */
3035                 switch (((extid >> 2) & 0x04) | (extid & 0x03)) {
3036                 case 0:
3037                         mtd->oobsize = 128;
3038                         break;
3039                 case 1:
3040                         mtd->oobsize = 224;
3041                         break;
3042                 case 2:
3043                         mtd->oobsize = 448;
3044                         break;
3045                 case 3:
3046                         mtd->oobsize = 64;
3047                         break;
3048                 case 4:
3049                         mtd->oobsize = 32;
3050                         break;
3051                 case 5:
3052                         mtd->oobsize = 16;
3053                         break;
3054                 default:
3055                         mtd->oobsize = 640;
3056                         break;
3057                 }
3058                 extid >>= 2;
3059                 /* Calc blocksize */
3060                 tmp = ((extid >> 1) & 0x04) | (extid & 0x03);
3061                 if (tmp < 0x03)
3062                         mtd->erasesize = (128 * 1024) << tmp;
3063                 else if (tmp == 0x03)
3064                         mtd->erasesize = 768 * 1024;
3065                 else
3066                         mtd->erasesize = (64 * 1024) << tmp;
3067                 *busw = 0;
3068         } else {
3069                 /* Calc pagesize */
3070                 mtd->writesize = 1024 << (extid & 0x03);
3071                 extid >>= 2;
3072                 /* Calc oobsize */
3073                 mtd->oobsize = (8 << (extid & 0x01)) *
3074                         (mtd->writesize >> 9);
3075                 extid >>= 2;
3076                 /* Calc blocksize. Blocksize is multiples of 64KiB */
3077                 mtd->erasesize = (64 * 1024) << (extid & 0x03);
3078                 extid >>= 2;
3079                 /* Get buswidth information */
3080                 *busw = (extid & 0x01) ? NAND_BUSWIDTH_16 : 0;
3081         }
3082 }
3083
3084 /*
3085  * Old devices have chip data hardcoded in the device ID table. nand_decode_id
3086  * decodes a matching ID table entry and assigns the MTD size parameters for
3087  * the chip.
3088  */
3089 static void nand_decode_id(struct mtd_info *mtd, struct nand_chip *chip,
3090                                 struct nand_flash_dev *type, u8 id_data[8],
3091                                 int *busw)
3092 {
3093         int maf_id = id_data[0];
3094
3095         mtd->erasesize = type->erasesize;
3096         mtd->writesize = type->pagesize;
3097         mtd->oobsize = mtd->writesize / 32;
3098         *busw = type->options & NAND_BUSWIDTH_16;
3099
3100         /*
3101          * Check for Spansion/AMD ID + repeating 5th, 6th byte since
3102          * some Spansion chips have erasesize that conflicts with size
3103          * listed in nand_ids table.
3104          * Data sheet (5 byte ID): Spansion S30ML-P ORNAND (p.39)
3105          */
3106         if (maf_id == NAND_MFR_AMD && id_data[4] != 0x00 && id_data[5] == 0x00
3107                         && id_data[6] == 0x00 && id_data[7] == 0x00
3108                         && mtd->writesize == 512) {
3109                 mtd->erasesize = 128 * 1024;
3110                 mtd->erasesize <<= ((id_data[3] & 0x03) << 1);
3111         }
3112 }
3113
3114 /*
3115  * Set the bad block marker/indicator (BBM/BBI) patterns according to some
3116  * heuristic patterns using various detected parameters (e.g., manufacturer,
3117  * page size, cell-type information).
3118  */
3119 static void nand_decode_bbm_options(struct mtd_info *mtd,
3120                                     struct nand_chip *chip, u8 id_data[8])
3121 {
3122         int maf_id = id_data[0];
3123
3124         /* Set the bad block position */
3125         if (mtd->writesize > 512 || (chip->options & NAND_BUSWIDTH_16))
3126                 chip->badblockpos = NAND_LARGE_BADBLOCK_POS;
3127         else
3128                 chip->badblockpos = NAND_SMALL_BADBLOCK_POS;
3129
3130         /*
3131          * Bad block marker is stored in the last page of each block on Samsung
3132          * and Hynix MLC devices; stored in first two pages of each block on
3133          * Micron devices with 2KiB pages and on SLC Samsung, Hynix, Toshiba,
3134          * AMD/Spansion, and Macronix.  All others scan only the first page.
3135          */
3136         if ((chip->cellinfo & NAND_CI_CELLTYPE_MSK) &&
3137                         (maf_id == NAND_MFR_SAMSUNG ||
3138                          maf_id == NAND_MFR_HYNIX))
3139                 chip->bbt_options |= NAND_BBT_SCANLASTPAGE;
3140         else if ((!(chip->cellinfo & NAND_CI_CELLTYPE_MSK) &&
3141                                 (maf_id == NAND_MFR_SAMSUNG ||
3142                                  maf_id == NAND_MFR_HYNIX ||
3143                                  maf_id == NAND_MFR_TOSHIBA ||
3144                                  maf_id == NAND_MFR_AMD ||
3145                                  maf_id == NAND_MFR_MACRONIX)) ||
3146                         (mtd->writesize == 2048 &&
3147                          maf_id == NAND_MFR_MICRON))
3148                 chip->bbt_options |= NAND_BBT_SCAN2NDPAGE;
3149 }
3150
3151 /*
3152  * Get the flash and manufacturer id and lookup if the type is supported.
3153  */
3154 static struct nand_flash_dev *nand_get_flash_type(struct mtd_info *mtd,
3155                                                   struct nand_chip *chip,
3156                                                   int busw,
3157                                                   int *maf_id, int *dev_id,
3158                                                   struct nand_flash_dev *type)
3159 {
3160         int i, maf_idx;
3161         u8 id_data[8];
3162
3163         /* Select the device */
3164         chip->select_chip(mtd, 0);
3165
3166         /*
3167          * Reset the chip, required by some chips (e.g. Micron MT29FxGxxxxx)
3168          * after power-up.
3169          */
3170         chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
3171
3172         /* Send the command for reading device ID */
3173         chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
3174
3175         /* Read manufacturer and device IDs */
3176         *maf_id = chip->read_byte(mtd);
3177         *dev_id = chip->read_byte(mtd);
3178
3179         /*
3180          * Try again to make sure, as some systems the bus-hold or other
3181          * interface concerns can cause random data which looks like a
3182          * possibly credible NAND flash to appear. If the two results do
3183          * not match, ignore the device completely.
3184          */
3185
3186         chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
3187
3188         /* Read entire ID string */
3189         for (i = 0; i < 8; i++)
3190                 id_data[i] = chip->read_byte(mtd);
3191
3192         if (id_data[0] != *maf_id || id_data[1] != *dev_id) {
3193                 pr_info("%s: second ID read did not match "
3194                         "%02x,%02x against %02x,%02x\n", __func__,
3195                         *maf_id, *dev_id, id_data[0], id_data[1]);
3196                 return ERR_PTR(-ENODEV);
3197         }
3198
3199         if (!type)
3200                 type = nand_flash_ids;
3201
3202         for (; type->name != NULL; type++)
3203                 if (*dev_id == type->id)
3204                         break;
3205
3206         chip->onfi_version = 0;
3207         if (!type->name || !type->pagesize) {
3208                 /* Check is chip is ONFI compliant */
3209                 if (nand_flash_detect_onfi(mtd, chip, &busw))
3210                         goto ident_done;
3211         }
3212
3213         if (!type->name)
3214                 return ERR_PTR(-ENODEV);
3215
3216         if (!mtd->name)
3217                 mtd->name = type->name;
3218
3219         chip->chipsize = (uint64_t)type->chipsize << 20;
3220
3221         if (!type->pagesize && chip->init_size) {
3222                 /* Set the pagesize, oobsize, erasesize by the driver */
3223                 busw = chip->init_size(mtd, chip, id_data);
3224         } else if (!type->pagesize) {
3225                 /* Decode parameters from extended ID */
3226                 nand_decode_ext_id(mtd, chip, id_data, &busw);
3227         } else {
3228                 nand_decode_id(mtd, chip, type, id_data, &busw);
3229         }
3230         /* Get chip options */
3231         chip->options |= type->options;
3232
3233         /*
3234          * Check if chip is not a Samsung device. Do not clear the
3235          * options for chips which do not have an extended id.
3236          */
3237         if (*maf_id != NAND_MFR_SAMSUNG && !type->pagesize)
3238                 chip->options &= ~NAND_SAMSUNG_LP_OPTIONS;
3239 ident_done:
3240
3241         /* Try to identify manufacturer */
3242         for (maf_idx = 0; nand_manuf_ids[maf_idx].id != 0x0; maf_idx++) {
3243                 if (nand_manuf_ids[maf_idx].id == *maf_id)
3244                         break;
3245         }
3246
3247         /*
3248          * Check, if buswidth is correct. Hardware drivers should set
3249          * chip correct!
3250          */
3251         if (busw != (chip->options & NAND_BUSWIDTH_16)) {
3252                 pr_info("NAND device: Manufacturer ID:"
3253                         " 0x%02x, Chip ID: 0x%02x (%s %s)\n", *maf_id,
3254                         *dev_id, nand_manuf_ids[maf_idx].name, mtd->name);
3255                 pr_warn("NAND bus width %d instead %d bit\n",
3256                            (chip->options & NAND_BUSWIDTH_16) ? 16 : 8,
3257                            busw ? 16 : 8);
3258                 return ERR_PTR(-EINVAL);
3259         }
3260
3261         nand_decode_bbm_options(mtd, chip, id_data);
3262
3263         /* Calculate the address shift from the page size */
3264         chip->page_shift = ffs(mtd->writesize) - 1;
3265         /* Convert chipsize to number of pages per chip -1 */
3266         chip->pagemask = (chip->chipsize >> chip->page_shift) - 1;
3267
3268         chip->bbt_erase_shift = chip->phys_erase_shift =
3269                 ffs(mtd->erasesize) - 1;
3270         if (chip->chipsize & 0xffffffff)
3271                 chip->chip_shift = ffs((unsigned)chip->chipsize) - 1;
3272         else {
3273                 chip->chip_shift = ffs((unsigned)(chip->chipsize >> 32));
3274                 chip->chip_shift += 32 - 1;
3275         }
3276
3277         chip->badblockbits = 8;
3278
3279         /* Check for AND chips with 4 page planes */
3280         if (chip->options & NAND_4PAGE_ARRAY)
3281                 chip->erase_cmd = multi_erase_cmd;
3282         else
3283                 chip->erase_cmd = single_erase_cmd;
3284
3285         /* Do not replace user supplied command function! */
3286         if (mtd->writesize > 512 && chip->cmdfunc == nand_command)
3287                 chip->cmdfunc = nand_command_lp;
3288
3289         pr_info("NAND device: Manufacturer ID: 0x%02x, Chip ID: 0x%02x (%s %s),"
3290                 " page size: %d, OOB size: %d\n",
3291                 *maf_id, *dev_id, nand_manuf_ids[maf_idx].name,
3292                 chip->onfi_version ? chip->onfi_params.model : type->name,
3293                 mtd->writesize, mtd->oobsize);
3294
3295         return type;
3296 }
3297
3298 /**
3299  * nand_scan_ident - [NAND Interface] Scan for the NAND device
3300  * @mtd: MTD device structure
3301  * @maxchips: number of chips to scan for
3302  * @table: alternative NAND ID table
3303  *
3304  * This is the first phase of the normal nand_scan() function. It reads the
3305  * flash ID and sets up MTD fields accordingly.
3306  *
3307  * The mtd->owner field must be set to the module of the caller.
3308  */
3309 int nand_scan_ident(struct mtd_info *mtd, int maxchips,
3310                     struct nand_flash_dev *table)
3311 {
3312         int i, busw, nand_maf_id, nand_dev_id;
3313         struct nand_chip *chip = mtd->priv;
3314         struct nand_flash_dev *type;
3315
3316         /* Get buswidth to select the correct functions */
3317         busw = chip->options & NAND_BUSWIDTH_16;
3318         /* Set the default functions */
3319         nand_set_defaults(chip, busw);
3320
3321         /* Read the flash type */
3322         type = nand_get_flash_type(mtd, chip, busw,
3323                                 &nand_maf_id, &nand_dev_id, table);
3324
3325         if (IS_ERR(type)) {
3326                 if (!(chip->options & NAND_SCAN_SILENT_NODEV))
3327                         pr_warn("No NAND device found\n");
3328                 chip->select_chip(mtd, -1);
3329                 return PTR_ERR(type);
3330         }
3331
3332         /* Check for a chip array */
3333         for (i = 1; i < maxchips; i++) {
3334                 chip->select_chip(mtd, i);
3335                 /* See comment in nand_get_flash_type for reset */
3336                 chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
3337                 /* Send the command for reading device ID */
3338                 chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
3339                 /* Read manufacturer and device IDs */
3340                 if (nand_maf_id != chip->read_byte(mtd) ||
3341                     nand_dev_id != chip->read_byte(mtd))
3342                         break;
3343         }
3344         if (i > 1)
3345                 pr_info("%d NAND chips detected\n", i);
3346
3347         /* Store the number of chips and calc total size for mtd */
3348         chip->numchips = i;
3349         mtd->size = i * chip->chipsize;
3350
3351         return 0;
3352 }
3353 EXPORT_SYMBOL(nand_scan_ident);
3354
3355
3356 /**
3357  * nand_scan_tail - [NAND Interface] Scan for the NAND device
3358  * @mtd: MTD device structure
3359  *
3360  * This is the second phase of the normal nand_scan() function. It fills out
3361  * all the uninitialized function pointers with the defaults and scans for a
3362  * bad block table if appropriate.
3363  */
3364 int nand_scan_tail(struct mtd_info *mtd)
3365 {
3366         int i;
3367         struct nand_chip *chip = mtd->priv;
3368
3369         /* New bad blocks should be marked in OOB, flash-based BBT, or both */
3370         BUG_ON((chip->bbt_options & NAND_BBT_NO_OOB_BBM) &&
3371                         !(chip->bbt_options & NAND_BBT_USE_FLASH));
3372
3373         if (!(chip->options & NAND_OWN_BUFFERS))
3374                 chip->buffers = kmalloc(sizeof(*chip->buffers), GFP_KERNEL);
3375         if (!chip->buffers)
3376                 return -ENOMEM;
3377
3378         /* Set the internal oob buffer location, just after the page data */
3379         chip->oob_poi = chip->buffers->databuf + mtd->writesize;
3380
3381         /*
3382          * If no default placement scheme is given, select an appropriate one.
3383          */
3384         if (!chip->ecc.layout && (chip->ecc.mode != NAND_ECC_SOFT_BCH)) {
3385                 switch (mtd->oobsize) {
3386                 case 8:
3387                         chip->ecc.layout = &nand_oob_8;
3388                         break;
3389                 case 16:
3390                         chip->ecc.layout = &nand_oob_16;
3391                         break;
3392                 case 64:
3393                         chip->ecc.layout = &nand_oob_64;
3394                         break;
3395                 case 128:
3396                         chip->ecc.layout = &nand_oob_128;
3397                         break;
3398                 default:
3399                         pr_warn("No oob scheme defined for oobsize %d\n",
3400                                    mtd->oobsize);
3401                         BUG();
3402                 }
3403         }
3404
3405         if (!chip->write_page)
3406                 chip->write_page = nand_write_page;
3407
3408         /* set for ONFI nand */
3409         if (!chip->onfi_set_features)
3410                 chip->onfi_set_features = nand_onfi_set_features;
3411         if (!chip->onfi_get_features)
3412                 chip->onfi_get_features = nand_onfi_get_features;
3413
3414         /*
3415          * Check ECC mode, default to software if 3byte/512byte hardware ECC is
3416          * selected and we have 256 byte pagesize fallback to software ECC
3417          */
3418
3419         switch (chip->ecc.mode) {
3420         case NAND_ECC_HW_OOB_FIRST:
3421                 /* Similar to NAND_ECC_HW, but a separate read_page handle */
3422                 if (!chip->ecc.calculate || !chip->ecc.correct ||
3423                      !chip->ecc.hwctl) {
3424                         pr_warn("No ECC functions supplied; "
3425                                    "hardware ECC not possible\n");
3426                         BUG();
3427                 }
3428                 if (!chip->ecc.read_page)
3429                         chip->ecc.read_page = nand_read_page_hwecc_oob_first;
3430
3431         case NAND_ECC_HW:
3432                 /* Use standard hwecc read page function? */
3433                 if (!chip->ecc.read_page)
3434                         chip->ecc.read_page = nand_read_page_hwecc;
3435                 if (!chip->ecc.write_page)
3436                         chip->ecc.write_page = nand_write_page_hwecc;
3437                 if (!chip->ecc.read_page_raw)
3438                         chip->ecc.read_page_raw = nand_read_page_raw;
3439                 if (!chip->ecc.write_page_raw)
3440                         chip->ecc.write_page_raw = nand_write_page_raw;
3441                 if (!chip->ecc.read_oob)
3442                         chip->ecc.read_oob = nand_read_oob_std;
3443                 if (!chip->ecc.write_oob)
3444                         chip->ecc.write_oob = nand_write_oob_std;
3445
3446         case NAND_ECC_HW_SYNDROME:
3447                 if ((!chip->ecc.calculate || !chip->ecc.correct ||
3448                      !chip->ecc.hwctl) &&
3449                     (!chip->ecc.read_page ||
3450                      chip->ecc.read_page == nand_read_page_hwecc ||
3451                      !chip->ecc.write_page ||
3452                      chip->ecc.write_page == nand_write_page_hwecc)) {
3453                         pr_warn("No ECC functions supplied; "
3454                                    "hardware ECC not possible\n");
3455                         BUG();
3456                 }
3457                 /* Use standard syndrome read/write page function? */
3458                 if (!chip->ecc.read_page)
3459                         chip->ecc.read_page = nand_read_page_syndrome;
3460                 if (!chip->ecc.write_page)
3461                         chip->ecc.write_page = nand_write_page_syndrome;
3462                 if (!chip->ecc.read_page_raw)
3463                         chip->ecc.read_page_raw = nand_read_page_raw_syndrome;
3464                 if (!chip->ecc.write_page_raw)
3465                         chip->ecc.write_page_raw = nand_write_page_raw_syndrome;
3466                 if (!chip->ecc.read_oob)
3467                         chip->ecc.read_oob = nand_read_oob_syndrome;
3468                 if (!chip->ecc.write_oob)
3469                         chip->ecc.write_oob = nand_write_oob_syndrome;
3470
3471                 if (mtd->writesize >= chip->ecc.size) {
3472                         if (!chip->ecc.strength) {
3473                                 pr_warn("Driver must set ecc.strength when using hardware ECC\n");
3474                                 BUG();
3475                         }
3476                         break;
3477                 }
3478                 pr_warn("%d byte HW ECC not possible on "
3479                            "%d byte page size, fallback to SW ECC\n",
3480                            chip->ecc.size, mtd->writesize);
3481                 chip->ecc.mode = NAND_ECC_SOFT;
3482
3483         case NAND_ECC_SOFT:
3484                 chip->ecc.calculate = nand_calculate_ecc;
3485                 chip->ecc.correct = nand_correct_data;
3486                 chip->ecc.read_page = nand_read_page_swecc;
3487                 chip->ecc.read_subpage = nand_read_subpage;
3488                 chip->ecc.write_page = nand_write_page_swecc;
3489                 chip->ecc.read_page_raw = nand_read_page_raw;
3490                 chip->ecc.write_page_raw = nand_write_page_raw;
3491                 chip->ecc.read_oob = nand_read_oob_std;
3492                 chip->ecc.write_oob = nand_write_oob_std;
3493                 if (!chip->ecc.size)
3494                         chip->ecc.size = 256;
3495                 chip->ecc.bytes = 3;
3496                 chip->ecc.strength = 1;
3497                 break;
3498
3499         case NAND_ECC_SOFT_BCH:
3500                 if (!mtd_nand_has_bch()) {
3501                         pr_warn("CONFIG_MTD_ECC_BCH not enabled\n");
3502                         BUG();
3503                 }
3504                 chip->ecc.calculate = nand_bch_calculate_ecc;
3505                 chip->ecc.correct = nand_bch_correct_data;
3506                 chip->ecc.read_page = nand_read_page_swecc;
3507                 chip->ecc.read_subpage = nand_read_subpage;
3508                 chip->ecc.write_page = nand_write_page_swecc;
3509                 chip->ecc.read_page_raw = nand_read_page_raw;
3510                 chip->ecc.write_page_raw = nand_write_page_raw;
3511                 chip->ecc.read_oob = nand_read_oob_std;
3512                 chip->ecc.write_oob = nand_write_oob_std;
3513                 /*
3514                  * Board driver should supply ecc.size and ecc.bytes values to
3515                  * select how many bits are correctable; see nand_bch_init()
3516                  * for details. Otherwise, default to 4 bits for large page
3517                  * devices.
3518                  */
3519                 if (!chip->ecc.size && (mtd->oobsize >= 64)) {
3520                         chip->ecc.size = 512;
3521                         chip->ecc.bytes = 7;
3522                 }
3523                 chip->ecc.priv = nand_bch_init(mtd,
3524                                                chip->ecc.size,
3525                                                chip->ecc.bytes,
3526                                                &chip->ecc.layout);
3527                 if (!chip->ecc.priv) {
3528                         pr_warn("BCH ECC initialization failed!\n");
3529                         BUG();
3530                 }
3531                 chip->ecc.strength =
3532                         chip->ecc.bytes * 8 / fls(8 * chip->ecc.size);
3533                 break;
3534
3535         case NAND_ECC_NONE:
3536                 pr_warn("NAND_ECC_NONE selected by board driver. "
3537                            "This is not recommended!\n");
3538                 chip->ecc.read_page = nand_read_page_raw;
3539                 chip->ecc.write_page = nand_write_page_raw;
3540                 chip->ecc.read_oob = nand_read_oob_std;
3541                 chip->ecc.read_page_raw = nand_read_page_raw;
3542                 chip->ecc.write_page_raw = nand_write_page_raw;
3543                 chip->ecc.write_oob = nand_write_oob_std;
3544                 chip->ecc.size = mtd->writesize;
3545                 chip->ecc.bytes = 0;
3546                 chip->ecc.strength = 0;
3547                 break;
3548
3549         default:
3550                 pr_warn("Invalid NAND_ECC_MODE %d\n", chip->ecc.mode);
3551                 BUG();
3552         }
3553
3554         /* For many systems, the standard OOB write also works for raw */
3555         if (!chip->ecc.read_oob_raw)
3556                 chip->ecc.read_oob_raw = chip->ecc.read_oob;
3557         if (!chip->ecc.write_oob_raw)
3558                 chip->ecc.write_oob_raw = chip->ecc.write_oob;
3559
3560         /*
3561          * The number of bytes available for a client to place data into
3562          * the out of band area.
3563          */
3564         chip->ecc.layout->oobavail = 0;
3565         for (i = 0; chip->ecc.layout->oobfree[i].length
3566                         && i < ARRAY_SIZE(chip->ecc.layout->oobfree); i++)
3567                 chip->ecc.layout->oobavail +=
3568                         chip->ecc.layout->oobfree[i].length;
3569         mtd->oobavail = chip->ecc.layout->oobavail;
3570
3571         /*
3572          * Set the number of read / write steps for one page depending on ECC
3573          * mode.
3574          */
3575         chip->ecc.steps = mtd->writesize / chip->ecc.size;
3576         if (chip->ecc.steps * chip->ecc.size != mtd->writesize) {
3577                 pr_warn("Invalid ECC parameters\n");
3578                 BUG();
3579         }
3580         chip->ecc.total = chip->ecc.steps * chip->ecc.bytes;
3581
3582         /* Allow subpage writes up to ecc.steps. Not possible for MLC flash */
3583         if (!(chip->options & NAND_NO_SUBPAGE_WRITE) &&
3584             !(chip->cellinfo & NAND_CI_CELLTYPE_MSK)) {
3585                 switch (chip->ecc.steps) {
3586                 case 2:
3587                         mtd->subpage_sft = 1;
3588                         break;
3589                 case 4:
3590                 case 8:
3591                 case 16:
3592                         mtd->subpage_sft = 2;
3593                         break;
3594                 }
3595         }
3596         chip->subpagesize = mtd->writesize >> mtd->subpage_sft;
3597
3598         /* Initialize state */
3599         chip->state = FL_READY;
3600
3601         /* De-select the device */
3602         chip->select_chip(mtd, -1);
3603
3604         /* Invalidate the pagebuffer reference */
3605         chip->pagebuf = -1;
3606
3607         /* Large page NAND with SOFT_ECC should support subpage reads */
3608         if ((chip->ecc.mode == NAND_ECC_SOFT) && (chip->page_shift > 9))
3609                 chip->options |= NAND_SUBPAGE_READ;
3610
3611         /* Fill in remaining MTD driver data */
3612         mtd->type = MTD_NANDFLASH;
3613         mtd->flags = (chip->options & NAND_ROM) ? MTD_CAP_ROM :
3614                                                 MTD_CAP_NANDFLASH;
3615         mtd->_erase = nand_erase;
3616         mtd->_point = NULL;
3617         mtd->_unpoint = NULL;
3618         mtd->_read = nand_read;
3619         mtd->_write = nand_write;
3620         mtd->_panic_write = panic_nand_write;
3621         mtd->_read_oob = nand_read_oob;
3622         mtd->_write_oob = nand_write_oob;
3623         mtd->_sync = nand_sync;
3624         mtd->_lock = NULL;
3625         mtd->_unlock = NULL;
3626         mtd->_suspend = nand_suspend;
3627         mtd->_resume = nand_resume;
3628         mtd->_block_isbad = nand_block_isbad;
3629         mtd->_block_markbad = nand_block_markbad;
3630         mtd->writebufsize = mtd->writesize;
3631
3632         /* propagate ecc info to mtd_info */
3633         mtd->ecclayout = chip->ecc.layout;
3634         mtd->ecc_strength = chip->ecc.strength;
3635         /*
3636          * Initialize bitflip_threshold to its default prior scan_bbt() call.
3637          * scan_bbt() might invoke mtd_read(), thus bitflip_threshold must be
3638          * properly set.
3639          */
3640         if (!mtd->bitflip_threshold)
3641                 mtd->bitflip_threshold = mtd->ecc_strength;
3642
3643         /* Check, if we should skip the bad block table scan */
3644         if (chip->options & NAND_SKIP_BBTSCAN)
3645                 return 0;
3646
3647         /* Build bad block table */
3648         return chip->scan_bbt(mtd);
3649 }
3650 EXPORT_SYMBOL(nand_scan_tail);
3651
3652 /*
3653  * is_module_text_address() isn't exported, and it's mostly a pointless
3654  * test if this is a module _anyway_ -- they'd have to try _really_ hard
3655  * to call us from in-kernel code if the core NAND support is modular.
3656  */
3657 #ifdef MODULE
3658 #define caller_is_module() (1)
3659 #else
3660 #define caller_is_module() \
3661         is_module_text_address((unsigned long)__builtin_return_address(0))
3662 #endif
3663
3664 /**
3665  * nand_scan - [NAND Interface] Scan for the NAND device
3666  * @mtd: MTD device structure
3667  * @maxchips: number of chips to scan for
3668  *
3669  * This fills out all the uninitialized function pointers with the defaults.
3670  * The flash ID is read and the mtd/chip structures are filled with the
3671  * appropriate values. The mtd->owner field must be set to the module of the
3672  * caller.
3673  */
3674 int nand_scan(struct mtd_info *mtd, int maxchips)
3675 {
3676         int ret;
3677
3678         /* Many callers got this wrong, so check for it for a while... */
3679         if (!mtd->owner && caller_is_module()) {
3680                 pr_crit("%s called with NULL mtd->owner!\n", __func__);
3681                 BUG();
3682         }
3683
3684         ret = nand_scan_ident(mtd, maxchips, NULL);
3685         if (!ret)
3686                 ret = nand_scan_tail(mtd);
3687         return ret;
3688 }
3689 EXPORT_SYMBOL(nand_scan);
3690
3691 /**
3692  * nand_release - [NAND Interface] Free resources held by the NAND device
3693  * @mtd: MTD device structure
3694  */
3695 void nand_release(struct mtd_info *mtd)
3696 {
3697         struct nand_chip *chip = mtd->priv;
3698
3699         if (chip->ecc.mode == NAND_ECC_SOFT_BCH)
3700                 nand_bch_free((struct nand_bch_control *)chip->ecc.priv);
3701
3702         mtd_device_unregister(mtd);
3703
3704         /* Free bad block table memory */
3705         kfree(chip->bbt);
3706         if (!(chip->options & NAND_OWN_BUFFERS))
3707                 kfree(chip->buffers);
3708
3709         /* Free bad block descriptor memory */
3710         if (chip->badblock_pattern && chip->badblock_pattern->options
3711                         & NAND_BBT_DYNAMICSTRUCT)
3712                 kfree(chip->badblock_pattern);
3713 }
3714 EXPORT_SYMBOL_GPL(nand_release);
3715
3716 static int __init nand_base_init(void)
3717 {
3718         led_trigger_register_simple("nand-disk", &nand_led_trigger);
3719         return 0;
3720 }
3721
3722 static void __exit nand_base_exit(void)
3723 {
3724         led_trigger_unregister_simple(nand_led_trigger);
3725 }
3726
3727 module_init(nand_base_init);
3728 module_exit(nand_base_exit);
3729
3730 MODULE_LICENSE("GPL");
3731 MODULE_AUTHOR("Steven J. Hill <sjhill@realitydiluted.com>");
3732 MODULE_AUTHOR("Thomas Gleixner <tglx@linutronix.de>");
3733 MODULE_DESCRIPTION("Generic NAND flash driver code");