lib/raid6: Add AVX2 optimized gen_syndrome functions
[linux-3.10.git] / lib / dma-debug.c
1 /*
2  * Copyright (C) 2008 Advanced Micro Devices, Inc.
3  *
4  * Author: Joerg Roedel <joerg.roedel@amd.com>
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License version 2 as published
8  * by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
18  */
19
20 #include <linux/scatterlist.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/stacktrace.h>
23 #include <linux/dma-debug.h>
24 #include <linux/spinlock.h>
25 #include <linux/debugfs.h>
26 #include <linux/uaccess.h>
27 #include <linux/export.h>
28 #include <linux/device.h>
29 #include <linux/types.h>
30 #include <linux/sched.h>
31 #include <linux/ctype.h>
32 #include <linux/list.h>
33 #include <linux/slab.h>
34
35 #include <asm/sections.h>
36
37 #define HASH_SIZE       1024ULL
38 #define HASH_FN_SHIFT   13
39 #define HASH_FN_MASK    (HASH_SIZE - 1)
40
41 enum {
42         dma_debug_single,
43         dma_debug_page,
44         dma_debug_sg,
45         dma_debug_coherent,
46 };
47
48 #define DMA_DEBUG_STACKTRACE_ENTRIES 5
49
50 struct dma_debug_entry {
51         struct list_head list;
52         struct device    *dev;
53         int              type;
54         phys_addr_t      paddr;
55         u64              dev_addr;
56         u64              size;
57         int              direction;
58         int              sg_call_ents;
59         int              sg_mapped_ents;
60 #ifdef CONFIG_STACKTRACE
61         struct           stack_trace stacktrace;
62         unsigned long    st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
63 #endif
64 };
65
66 typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
67
68 struct hash_bucket {
69         struct list_head list;
70         spinlock_t lock;
71 } ____cacheline_aligned_in_smp;
72
73 /* Hash list to save the allocated dma addresses */
74 static struct hash_bucket dma_entry_hash[HASH_SIZE];
75 /* List of pre-allocated dma_debug_entry's */
76 static LIST_HEAD(free_entries);
77 /* Lock for the list above */
78 static DEFINE_SPINLOCK(free_entries_lock);
79
80 /* Global disable flag - will be set in case of an error */
81 static u32 global_disable __read_mostly;
82
83 /* Global error count */
84 static u32 error_count;
85
86 /* Global error show enable*/
87 static u32 show_all_errors __read_mostly;
88 /* Number of errors to show */
89 static u32 show_num_errors = 1;
90
91 static u32 num_free_entries;
92 static u32 min_free_entries;
93 static u32 nr_total_entries;
94
95 /* number of preallocated entries requested by kernel cmdline */
96 static u32 req_entries;
97
98 /* debugfs dentry's for the stuff above */
99 static struct dentry *dma_debug_dent        __read_mostly;
100 static struct dentry *global_disable_dent   __read_mostly;
101 static struct dentry *error_count_dent      __read_mostly;
102 static struct dentry *show_all_errors_dent  __read_mostly;
103 static struct dentry *show_num_errors_dent  __read_mostly;
104 static struct dentry *num_free_entries_dent __read_mostly;
105 static struct dentry *min_free_entries_dent __read_mostly;
106 static struct dentry *filter_dent           __read_mostly;
107
108 /* per-driver filter related state */
109
110 #define NAME_MAX_LEN    64
111
112 static char                  current_driver_name[NAME_MAX_LEN] __read_mostly;
113 static struct device_driver *current_driver                    __read_mostly;
114
115 static DEFINE_RWLOCK(driver_name_lock);
116
117 static const char *type2name[4] = { "single", "page",
118                                     "scather-gather", "coherent" };
119
120 static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
121                                    "DMA_FROM_DEVICE", "DMA_NONE" };
122
123 /*
124  * The access to some variables in this macro is racy. We can't use atomic_t
125  * here because all these variables are exported to debugfs. Some of them even
126  * writeable. This is also the reason why a lock won't help much. But anyway,
127  * the races are no big deal. Here is why:
128  *
129  *   error_count: the addition is racy, but the worst thing that can happen is
130  *                that we don't count some errors
131  *   show_num_errors: the subtraction is racy. Also no big deal because in
132  *                    worst case this will result in one warning more in the
133  *                    system log than the user configured. This variable is
134  *                    writeable via debugfs.
135  */
136 static inline void dump_entry_trace(struct dma_debug_entry *entry)
137 {
138 #ifdef CONFIG_STACKTRACE
139         if (entry) {
140                 pr_warning("Mapped at:\n");
141                 print_stack_trace(&entry->stacktrace, 0);
142         }
143 #endif
144 }
145
146 static bool driver_filter(struct device *dev)
147 {
148         struct device_driver *drv;
149         unsigned long flags;
150         bool ret;
151
152         /* driver filter off */
153         if (likely(!current_driver_name[0]))
154                 return true;
155
156         /* driver filter on and initialized */
157         if (current_driver && dev && dev->driver == current_driver)
158                 return true;
159
160         /* driver filter on, but we can't filter on a NULL device... */
161         if (!dev)
162                 return false;
163
164         if (current_driver || !current_driver_name[0])
165                 return false;
166
167         /* driver filter on but not yet initialized */
168         drv = dev->driver;
169         if (!drv)
170                 return false;
171
172         /* lock to protect against change of current_driver_name */
173         read_lock_irqsave(&driver_name_lock, flags);
174
175         ret = false;
176         if (drv->name &&
177             strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
178                 current_driver = drv;
179                 ret = true;
180         }
181
182         read_unlock_irqrestore(&driver_name_lock, flags);
183
184         return ret;
185 }
186
187 #define err_printk(dev, entry, format, arg...) do {                     \
188                 error_count += 1;                                       \
189                 if (driver_filter(dev) &&                               \
190                     (show_all_errors || show_num_errors > 0)) {         \
191                         WARN(1, "%s %s: " format,                       \
192                              dev ? dev_driver_string(dev) : "NULL",     \
193                              dev ? dev_name(dev) : "NULL", ## arg);     \
194                         dump_entry_trace(entry);                        \
195                 }                                                       \
196                 if (!show_all_errors && show_num_errors > 0)            \
197                         show_num_errors -= 1;                           \
198         } while (0);
199
200 /*
201  * Hash related functions
202  *
203  * Every DMA-API request is saved into a struct dma_debug_entry. To
204  * have quick access to these structs they are stored into a hash.
205  */
206 static int hash_fn(struct dma_debug_entry *entry)
207 {
208         /*
209          * Hash function is based on the dma address.
210          * We use bits 20-27 here as the index into the hash
211          */
212         return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
213 }
214
215 /*
216  * Request exclusive access to a hash bucket for a given dma_debug_entry.
217  */
218 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
219                                            unsigned long *flags)
220 {
221         int idx = hash_fn(entry);
222         unsigned long __flags;
223
224         spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
225         *flags = __flags;
226         return &dma_entry_hash[idx];
227 }
228
229 /*
230  * Give up exclusive access to the hash bucket
231  */
232 static void put_hash_bucket(struct hash_bucket *bucket,
233                             unsigned long *flags)
234 {
235         unsigned long __flags = *flags;
236
237         spin_unlock_irqrestore(&bucket->lock, __flags);
238 }
239
240 static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
241 {
242         return ((a->dev_addr == b->dev_addr) &&
243                 (a->dev == b->dev)) ? true : false;
244 }
245
246 static bool containing_match(struct dma_debug_entry *a,
247                              struct dma_debug_entry *b)
248 {
249         if (a->dev != b->dev)
250                 return false;
251
252         if ((b->dev_addr <= a->dev_addr) &&
253             ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))
254                 return true;
255
256         return false;
257 }
258
259 /*
260  * Search a given entry in the hash bucket list
261  */
262 static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,
263                                                   struct dma_debug_entry *ref,
264                                                   match_fn match)
265 {
266         struct dma_debug_entry *entry, *ret = NULL;
267         int matches = 0, match_lvl, last_lvl = 0;
268
269         list_for_each_entry(entry, &bucket->list, list) {
270                 if (!match(ref, entry))
271                         continue;
272
273                 /*
274                  * Some drivers map the same physical address multiple
275                  * times. Without a hardware IOMMU this results in the
276                  * same device addresses being put into the dma-debug
277                  * hash multiple times too. This can result in false
278                  * positives being reported. Therefore we implement a
279                  * best-fit algorithm here which returns the entry from
280                  * the hash which fits best to the reference value
281                  * instead of the first-fit.
282                  */
283                 matches += 1;
284                 match_lvl = 0;
285                 entry->size         == ref->size         ? ++match_lvl : 0;
286                 entry->type         == ref->type         ? ++match_lvl : 0;
287                 entry->direction    == ref->direction    ? ++match_lvl : 0;
288                 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
289
290                 if (match_lvl == 4) {
291                         /* perfect-fit - return the result */
292                         return entry;
293                 } else if (match_lvl > last_lvl) {
294                         /*
295                          * We found an entry that fits better then the
296                          * previous one
297                          */
298                         last_lvl = match_lvl;
299                         ret      = entry;
300                 }
301         }
302
303         /*
304          * If we have multiple matches but no perfect-fit, just return
305          * NULL.
306          */
307         ret = (matches == 1) ? ret : NULL;
308
309         return ret;
310 }
311
312 static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,
313                                                  struct dma_debug_entry *ref)
314 {
315         return __hash_bucket_find(bucket, ref, exact_match);
316 }
317
318 static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,
319                                                    struct dma_debug_entry *ref,
320                                                    unsigned long *flags)
321 {
322
323         unsigned int max_range = dma_get_max_seg_size(ref->dev);
324         struct dma_debug_entry *entry, index = *ref;
325         unsigned int range = 0;
326
327         while (range <= max_range) {
328                 entry = __hash_bucket_find(*bucket, &index, containing_match);
329
330                 if (entry)
331                         return entry;
332
333                 /*
334                  * Nothing found, go back a hash bucket
335                  */
336                 put_hash_bucket(*bucket, flags);
337                 range          += (1 << HASH_FN_SHIFT);
338                 index.dev_addr -= (1 << HASH_FN_SHIFT);
339                 *bucket = get_hash_bucket(&index, flags);
340         }
341
342         return NULL;
343 }
344
345 /*
346  * Add an entry to a hash bucket
347  */
348 static void hash_bucket_add(struct hash_bucket *bucket,
349                             struct dma_debug_entry *entry)
350 {
351         list_add_tail(&entry->list, &bucket->list);
352 }
353
354 /*
355  * Remove entry from a hash bucket list
356  */
357 static void hash_bucket_del(struct dma_debug_entry *entry)
358 {
359         list_del(&entry->list);
360 }
361
362 /*
363  * Dump mapping entries for debugging purposes
364  */
365 void debug_dma_dump_mappings(struct device *dev)
366 {
367         int idx;
368
369         for (idx = 0; idx < HASH_SIZE; idx++) {
370                 struct hash_bucket *bucket = &dma_entry_hash[idx];
371                 struct dma_debug_entry *entry;
372                 unsigned long flags;
373
374                 spin_lock_irqsave(&bucket->lock, flags);
375
376                 list_for_each_entry(entry, &bucket->list, list) {
377                         if (!dev || dev == entry->dev) {
378                                 dev_info(entry->dev,
379                                          "%s idx %d P=%Lx D=%Lx L=%Lx %s\n",
380                                          type2name[entry->type], idx,
381                                          (unsigned long long)entry->paddr,
382                                          entry->dev_addr, entry->size,
383                                          dir2name[entry->direction]);
384                         }
385                 }
386
387                 spin_unlock_irqrestore(&bucket->lock, flags);
388         }
389 }
390 EXPORT_SYMBOL(debug_dma_dump_mappings);
391
392 /*
393  * Wrapper function for adding an entry to the hash.
394  * This function takes care of locking itself.
395  */
396 static void add_dma_entry(struct dma_debug_entry *entry)
397 {
398         struct hash_bucket *bucket;
399         unsigned long flags;
400
401         bucket = get_hash_bucket(entry, &flags);
402         hash_bucket_add(bucket, entry);
403         put_hash_bucket(bucket, &flags);
404 }
405
406 static struct dma_debug_entry *__dma_entry_alloc(void)
407 {
408         struct dma_debug_entry *entry;
409
410         entry = list_entry(free_entries.next, struct dma_debug_entry, list);
411         list_del(&entry->list);
412         memset(entry, 0, sizeof(*entry));
413
414         num_free_entries -= 1;
415         if (num_free_entries < min_free_entries)
416                 min_free_entries = num_free_entries;
417
418         return entry;
419 }
420
421 /* struct dma_entry allocator
422  *
423  * The next two functions implement the allocator for
424  * struct dma_debug_entries.
425  */
426 static struct dma_debug_entry *dma_entry_alloc(void)
427 {
428         struct dma_debug_entry *entry;
429         unsigned long flags;
430
431         spin_lock_irqsave(&free_entries_lock, flags);
432
433         if (list_empty(&free_entries)) {
434                 pr_err("DMA-API: debugging out of memory - disabling\n");
435                 global_disable = true;
436                 spin_unlock_irqrestore(&free_entries_lock, flags);
437                 return NULL;
438         }
439
440         entry = __dma_entry_alloc();
441
442         spin_unlock_irqrestore(&free_entries_lock, flags);
443
444 #ifdef CONFIG_STACKTRACE
445         entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
446         entry->stacktrace.entries = entry->st_entries;
447         entry->stacktrace.skip = 2;
448         save_stack_trace(&entry->stacktrace);
449 #endif
450
451         return entry;
452 }
453
454 static void dma_entry_free(struct dma_debug_entry *entry)
455 {
456         unsigned long flags;
457
458         /*
459          * add to beginning of the list - this way the entries are
460          * more likely cache hot when they are reallocated.
461          */
462         spin_lock_irqsave(&free_entries_lock, flags);
463         list_add(&entry->list, &free_entries);
464         num_free_entries += 1;
465         spin_unlock_irqrestore(&free_entries_lock, flags);
466 }
467
468 int dma_debug_resize_entries(u32 num_entries)
469 {
470         int i, delta, ret = 0;
471         unsigned long flags;
472         struct dma_debug_entry *entry;
473         LIST_HEAD(tmp);
474
475         spin_lock_irqsave(&free_entries_lock, flags);
476
477         if (nr_total_entries < num_entries) {
478                 delta = num_entries - nr_total_entries;
479
480                 spin_unlock_irqrestore(&free_entries_lock, flags);
481
482                 for (i = 0; i < delta; i++) {
483                         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
484                         if (!entry)
485                                 break;
486
487                         list_add_tail(&entry->list, &tmp);
488                 }
489
490                 spin_lock_irqsave(&free_entries_lock, flags);
491
492                 list_splice(&tmp, &free_entries);
493                 nr_total_entries += i;
494                 num_free_entries += i;
495         } else {
496                 delta = nr_total_entries - num_entries;
497
498                 for (i = 0; i < delta && !list_empty(&free_entries); i++) {
499                         entry = __dma_entry_alloc();
500                         kfree(entry);
501                 }
502
503                 nr_total_entries -= i;
504         }
505
506         if (nr_total_entries != num_entries)
507                 ret = 1;
508
509         spin_unlock_irqrestore(&free_entries_lock, flags);
510
511         return ret;
512 }
513 EXPORT_SYMBOL(dma_debug_resize_entries);
514
515 /*
516  * DMA-API debugging init code
517  *
518  * The init code does two things:
519  *   1. Initialize core data structures
520  *   2. Preallocate a given number of dma_debug_entry structs
521  */
522
523 static int prealloc_memory(u32 num_entries)
524 {
525         struct dma_debug_entry *entry, *next_entry;
526         int i;
527
528         for (i = 0; i < num_entries; ++i) {
529                 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
530                 if (!entry)
531                         goto out_err;
532
533                 list_add_tail(&entry->list, &free_entries);
534         }
535
536         num_free_entries = num_entries;
537         min_free_entries = num_entries;
538
539         pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
540
541         return 0;
542
543 out_err:
544
545         list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
546                 list_del(&entry->list);
547                 kfree(entry);
548         }
549
550         return -ENOMEM;
551 }
552
553 static ssize_t filter_read(struct file *file, char __user *user_buf,
554                            size_t count, loff_t *ppos)
555 {
556         char buf[NAME_MAX_LEN + 1];
557         unsigned long flags;
558         int len;
559
560         if (!current_driver_name[0])
561                 return 0;
562
563         /*
564          * We can't copy to userspace directly because current_driver_name can
565          * only be read under the driver_name_lock with irqs disabled. So
566          * create a temporary copy first.
567          */
568         read_lock_irqsave(&driver_name_lock, flags);
569         len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
570         read_unlock_irqrestore(&driver_name_lock, flags);
571
572         return simple_read_from_buffer(user_buf, count, ppos, buf, len);
573 }
574
575 static ssize_t filter_write(struct file *file, const char __user *userbuf,
576                             size_t count, loff_t *ppos)
577 {
578         char buf[NAME_MAX_LEN];
579         unsigned long flags;
580         size_t len;
581         int i;
582
583         /*
584          * We can't copy from userspace directly. Access to
585          * current_driver_name is protected with a write_lock with irqs
586          * disabled. Since copy_from_user can fault and may sleep we
587          * need to copy to temporary buffer first
588          */
589         len = min(count, (size_t)(NAME_MAX_LEN - 1));
590         if (copy_from_user(buf, userbuf, len))
591                 return -EFAULT;
592
593         buf[len] = 0;
594
595         write_lock_irqsave(&driver_name_lock, flags);
596
597         /*
598          * Now handle the string we got from userspace very carefully.
599          * The rules are:
600          *         - only use the first token we got
601          *         - token delimiter is everything looking like a space
602          *           character (' ', '\n', '\t' ...)
603          *
604          */
605         if (!isalnum(buf[0])) {
606                 /*
607                  * If the first character userspace gave us is not
608                  * alphanumerical then assume the filter should be
609                  * switched off.
610                  */
611                 if (current_driver_name[0])
612                         pr_info("DMA-API: switching off dma-debug driver filter\n");
613                 current_driver_name[0] = 0;
614                 current_driver = NULL;
615                 goto out_unlock;
616         }
617
618         /*
619          * Now parse out the first token and use it as the name for the
620          * driver to filter for.
621          */
622         for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
623                 current_driver_name[i] = buf[i];
624                 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
625                         break;
626         }
627         current_driver_name[i] = 0;
628         current_driver = NULL;
629
630         pr_info("DMA-API: enable driver filter for driver [%s]\n",
631                 current_driver_name);
632
633 out_unlock:
634         write_unlock_irqrestore(&driver_name_lock, flags);
635
636         return count;
637 }
638
639 static const struct file_operations filter_fops = {
640         .read  = filter_read,
641         .write = filter_write,
642         .llseek = default_llseek,
643 };
644
645 static int dma_debug_fs_init(void)
646 {
647         dma_debug_dent = debugfs_create_dir("dma-api", NULL);
648         if (!dma_debug_dent) {
649                 pr_err("DMA-API: can not create debugfs directory\n");
650                 return -ENOMEM;
651         }
652
653         global_disable_dent = debugfs_create_bool("disabled", 0444,
654                         dma_debug_dent,
655                         &global_disable);
656         if (!global_disable_dent)
657                 goto out_err;
658
659         error_count_dent = debugfs_create_u32("error_count", 0444,
660                         dma_debug_dent, &error_count);
661         if (!error_count_dent)
662                 goto out_err;
663
664         show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
665                         dma_debug_dent,
666                         &show_all_errors);
667         if (!show_all_errors_dent)
668                 goto out_err;
669
670         show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
671                         dma_debug_dent,
672                         &show_num_errors);
673         if (!show_num_errors_dent)
674                 goto out_err;
675
676         num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
677                         dma_debug_dent,
678                         &num_free_entries);
679         if (!num_free_entries_dent)
680                 goto out_err;
681
682         min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
683                         dma_debug_dent,
684                         &min_free_entries);
685         if (!min_free_entries_dent)
686                 goto out_err;
687
688         filter_dent = debugfs_create_file("driver_filter", 0644,
689                                           dma_debug_dent, NULL, &filter_fops);
690         if (!filter_dent)
691                 goto out_err;
692
693         return 0;
694
695 out_err:
696         debugfs_remove_recursive(dma_debug_dent);
697
698         return -ENOMEM;
699 }
700
701 static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
702 {
703         struct dma_debug_entry *entry;
704         unsigned long flags;
705         int count = 0, i;
706
707         local_irq_save(flags);
708
709         for (i = 0; i < HASH_SIZE; ++i) {
710                 spin_lock(&dma_entry_hash[i].lock);
711                 list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
712                         if (entry->dev == dev) {
713                                 count += 1;
714                                 *out_entry = entry;
715                         }
716                 }
717                 spin_unlock(&dma_entry_hash[i].lock);
718         }
719
720         local_irq_restore(flags);
721
722         return count;
723 }
724
725 static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
726 {
727         struct device *dev = data;
728         struct dma_debug_entry *uninitialized_var(entry);
729         int count;
730
731         if (global_disable)
732                 return 0;
733
734         switch (action) {
735         case BUS_NOTIFY_UNBOUND_DRIVER:
736                 count = device_dma_allocations(dev, &entry);
737                 if (count == 0)
738                         break;
739                 err_printk(dev, entry, "DMA-API: device driver has pending "
740                                 "DMA allocations while released from device "
741                                 "[count=%d]\n"
742                                 "One of leaked entries details: "
743                                 "[device address=0x%016llx] [size=%llu bytes] "
744                                 "[mapped with %s] [mapped as %s]\n",
745                         count, entry->dev_addr, entry->size,
746                         dir2name[entry->direction], type2name[entry->type]);
747                 break;
748         default:
749                 break;
750         }
751
752         return 0;
753 }
754
755 void dma_debug_add_bus(struct bus_type *bus)
756 {
757         struct notifier_block *nb;
758
759         if (global_disable)
760                 return;
761
762         nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
763         if (nb == NULL) {
764                 pr_err("dma_debug_add_bus: out of memory\n");
765                 return;
766         }
767
768         nb->notifier_call = dma_debug_device_change;
769
770         bus_register_notifier(bus, nb);
771 }
772
773 /*
774  * Let the architectures decide how many entries should be preallocated.
775  */
776 void dma_debug_init(u32 num_entries)
777 {
778         int i;
779
780         if (global_disable)
781                 return;
782
783         for (i = 0; i < HASH_SIZE; ++i) {
784                 INIT_LIST_HEAD(&dma_entry_hash[i].list);
785                 spin_lock_init(&dma_entry_hash[i].lock);
786         }
787
788         if (dma_debug_fs_init() != 0) {
789                 pr_err("DMA-API: error creating debugfs entries - disabling\n");
790                 global_disable = true;
791
792                 return;
793         }
794
795         if (req_entries)
796                 num_entries = req_entries;
797
798         if (prealloc_memory(num_entries) != 0) {
799                 pr_err("DMA-API: debugging out of memory error - disabled\n");
800                 global_disable = true;
801
802                 return;
803         }
804
805         nr_total_entries = num_free_entries;
806
807         pr_info("DMA-API: debugging enabled by kernel config\n");
808 }
809
810 static __init int dma_debug_cmdline(char *str)
811 {
812         if (!str)
813                 return -EINVAL;
814
815         if (strncmp(str, "off", 3) == 0) {
816                 pr_info("DMA-API: debugging disabled on kernel command line\n");
817                 global_disable = true;
818         }
819
820         return 0;
821 }
822
823 static __init int dma_debug_entries_cmdline(char *str)
824 {
825         int res;
826
827         if (!str)
828                 return -EINVAL;
829
830         res = get_option(&str, &req_entries);
831
832         if (!res)
833                 req_entries = 0;
834
835         return 0;
836 }
837
838 __setup("dma_debug=", dma_debug_cmdline);
839 __setup("dma_debug_entries=", dma_debug_entries_cmdline);
840
841 static void check_unmap(struct dma_debug_entry *ref)
842 {
843         struct dma_debug_entry *entry;
844         struct hash_bucket *bucket;
845         unsigned long flags;
846
847         if (dma_mapping_error(ref->dev, ref->dev_addr)) {
848                 err_printk(ref->dev, NULL, "DMA-API: device driver tries "
849                            "to free an invalid DMA memory address\n");
850                 return;
851         }
852
853         bucket = get_hash_bucket(ref, &flags);
854         entry = bucket_find_exact(bucket, ref);
855
856         if (!entry) {
857                 err_printk(ref->dev, NULL, "DMA-API: device driver tries "
858                            "to free DMA memory it has not allocated "
859                            "[device address=0x%016llx] [size=%llu bytes]\n",
860                            ref->dev_addr, ref->size);
861                 goto out;
862         }
863
864         if (ref->size != entry->size) {
865                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
866                            "DMA memory with different size "
867                            "[device address=0x%016llx] [map size=%llu bytes] "
868                            "[unmap size=%llu bytes]\n",
869                            ref->dev_addr, entry->size, ref->size);
870         }
871
872         if (ref->type != entry->type) {
873                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
874                            "DMA memory with wrong function "
875                            "[device address=0x%016llx] [size=%llu bytes] "
876                            "[mapped as %s] [unmapped as %s]\n",
877                            ref->dev_addr, ref->size,
878                            type2name[entry->type], type2name[ref->type]);
879         } else if ((entry->type == dma_debug_coherent) &&
880                    (ref->paddr != entry->paddr)) {
881                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
882                            "DMA memory with different CPU address "
883                            "[device address=0x%016llx] [size=%llu bytes] "
884                            "[cpu alloc address=0x%016llx] "
885                            "[cpu free address=0x%016llx]",
886                            ref->dev_addr, ref->size,
887                            (unsigned long long)entry->paddr,
888                            (unsigned long long)ref->paddr);
889         }
890
891         if (ref->sg_call_ents && ref->type == dma_debug_sg &&
892             ref->sg_call_ents != entry->sg_call_ents) {
893                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
894                            "DMA sg list with different entry count "
895                            "[map count=%d] [unmap count=%d]\n",
896                            entry->sg_call_ents, ref->sg_call_ents);
897         }
898
899         /*
900          * This may be no bug in reality - but most implementations of the
901          * DMA API don't handle this properly, so check for it here
902          */
903         if (ref->direction != entry->direction) {
904                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
905                            "DMA memory with different direction "
906                            "[device address=0x%016llx] [size=%llu bytes] "
907                            "[mapped with %s] [unmapped with %s]\n",
908                            ref->dev_addr, ref->size,
909                            dir2name[entry->direction],
910                            dir2name[ref->direction]);
911         }
912
913         hash_bucket_del(entry);
914         dma_entry_free(entry);
915
916 out:
917         put_hash_bucket(bucket, &flags);
918 }
919
920 static void check_for_stack(struct device *dev, void *addr)
921 {
922         if (object_is_on_stack(addr))
923                 err_printk(dev, NULL, "DMA-API: device driver maps memory from"
924                                 "stack [addr=%p]\n", addr);
925 }
926
927 static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
928 {
929         unsigned long a1 = (unsigned long)addr;
930         unsigned long b1 = a1 + len;
931         unsigned long a2 = (unsigned long)start;
932         unsigned long b2 = (unsigned long)end;
933
934         return !(b1 <= a2 || a1 >= b2);
935 }
936
937 static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
938 {
939         if (overlap(addr, len, _text, _etext) ||
940             overlap(addr, len, __start_rodata, __end_rodata))
941                 err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
942 }
943
944 static void check_sync(struct device *dev,
945                        struct dma_debug_entry *ref,
946                        bool to_cpu)
947 {
948         struct dma_debug_entry *entry;
949         struct hash_bucket *bucket;
950         unsigned long flags;
951
952         bucket = get_hash_bucket(ref, &flags);
953
954         entry = bucket_find_contain(&bucket, ref, &flags);
955
956         if (!entry) {
957                 err_printk(dev, NULL, "DMA-API: device driver tries "
958                                 "to sync DMA memory it has not allocated "
959                                 "[device address=0x%016llx] [size=%llu bytes]\n",
960                                 (unsigned long long)ref->dev_addr, ref->size);
961                 goto out;
962         }
963
964         if (ref->size > entry->size) {
965                 err_printk(dev, entry, "DMA-API: device driver syncs"
966                                 " DMA memory outside allocated range "
967                                 "[device address=0x%016llx] "
968                                 "[allocation size=%llu bytes] "
969                                 "[sync offset+size=%llu]\n",
970                                 entry->dev_addr, entry->size,
971                                 ref->size);
972         }
973
974         if (entry->direction == DMA_BIDIRECTIONAL)
975                 goto out;
976
977         if (ref->direction != entry->direction) {
978                 err_printk(dev, entry, "DMA-API: device driver syncs "
979                                 "DMA memory with different direction "
980                                 "[device address=0x%016llx] [size=%llu bytes] "
981                                 "[mapped with %s] [synced with %s]\n",
982                                 (unsigned long long)ref->dev_addr, entry->size,
983                                 dir2name[entry->direction],
984                                 dir2name[ref->direction]);
985         }
986
987         if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
988                       !(ref->direction == DMA_TO_DEVICE))
989                 err_printk(dev, entry, "DMA-API: device driver syncs "
990                                 "device read-only DMA memory for cpu "
991                                 "[device address=0x%016llx] [size=%llu bytes] "
992                                 "[mapped with %s] [synced with %s]\n",
993                                 (unsigned long long)ref->dev_addr, entry->size,
994                                 dir2name[entry->direction],
995                                 dir2name[ref->direction]);
996
997         if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
998                        !(ref->direction == DMA_FROM_DEVICE))
999                 err_printk(dev, entry, "DMA-API: device driver syncs "
1000                                 "device write-only DMA memory to device "
1001                                 "[device address=0x%016llx] [size=%llu bytes] "
1002                                 "[mapped with %s] [synced with %s]\n",
1003                                 (unsigned long long)ref->dev_addr, entry->size,
1004                                 dir2name[entry->direction],
1005                                 dir2name[ref->direction]);
1006
1007 out:
1008         put_hash_bucket(bucket, &flags);
1009 }
1010
1011 void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
1012                         size_t size, int direction, dma_addr_t dma_addr,
1013                         bool map_single)
1014 {
1015         struct dma_debug_entry *entry;
1016
1017         if (unlikely(global_disable))
1018                 return;
1019
1020         if (unlikely(dma_mapping_error(dev, dma_addr)))
1021                 return;
1022
1023         entry = dma_entry_alloc();
1024         if (!entry)
1025                 return;
1026
1027         entry->dev       = dev;
1028         entry->type      = dma_debug_page;
1029         entry->paddr     = page_to_phys(page) + offset;
1030         entry->dev_addr  = dma_addr;
1031         entry->size      = size;
1032         entry->direction = direction;
1033
1034         if (map_single)
1035                 entry->type = dma_debug_single;
1036
1037         if (!PageHighMem(page)) {
1038                 void *addr = page_address(page) + offset;
1039
1040                 check_for_stack(dev, addr);
1041                 check_for_illegal_area(dev, addr, size);
1042         }
1043
1044         add_dma_entry(entry);
1045 }
1046 EXPORT_SYMBOL(debug_dma_map_page);
1047
1048 void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1049                           size_t size, int direction, bool map_single)
1050 {
1051         struct dma_debug_entry ref = {
1052                 .type           = dma_debug_page,
1053                 .dev            = dev,
1054                 .dev_addr       = addr,
1055                 .size           = size,
1056                 .direction      = direction,
1057         };
1058
1059         if (unlikely(global_disable))
1060                 return;
1061
1062         if (map_single)
1063                 ref.type = dma_debug_single;
1064
1065         check_unmap(&ref);
1066 }
1067 EXPORT_SYMBOL(debug_dma_unmap_page);
1068
1069 void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1070                       int nents, int mapped_ents, int direction)
1071 {
1072         struct dma_debug_entry *entry;
1073         struct scatterlist *s;
1074         int i;
1075
1076         if (unlikely(global_disable))
1077                 return;
1078
1079         for_each_sg(sg, s, mapped_ents, i) {
1080                 entry = dma_entry_alloc();
1081                 if (!entry)
1082                         return;
1083
1084                 entry->type           = dma_debug_sg;
1085                 entry->dev            = dev;
1086                 entry->paddr          = sg_phys(s);
1087                 entry->size           = sg_dma_len(s);
1088                 entry->dev_addr       = sg_dma_address(s);
1089                 entry->direction      = direction;
1090                 entry->sg_call_ents   = nents;
1091                 entry->sg_mapped_ents = mapped_ents;
1092
1093                 if (!PageHighMem(sg_page(s))) {
1094                         check_for_stack(dev, sg_virt(s));
1095                         check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
1096                 }
1097
1098                 add_dma_entry(entry);
1099         }
1100 }
1101 EXPORT_SYMBOL(debug_dma_map_sg);
1102
1103 static int get_nr_mapped_entries(struct device *dev,
1104                                  struct dma_debug_entry *ref)
1105 {
1106         struct dma_debug_entry *entry;
1107         struct hash_bucket *bucket;
1108         unsigned long flags;
1109         int mapped_ents;
1110
1111         bucket       = get_hash_bucket(ref, &flags);
1112         entry        = bucket_find_exact(bucket, ref);
1113         mapped_ents  = 0;
1114
1115         if (entry)
1116                 mapped_ents = entry->sg_mapped_ents;
1117         put_hash_bucket(bucket, &flags);
1118
1119         return mapped_ents;
1120 }
1121
1122 void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1123                         int nelems, int dir)
1124 {
1125         struct scatterlist *s;
1126         int mapped_ents = 0, i;
1127
1128         if (unlikely(global_disable))
1129                 return;
1130
1131         for_each_sg(sglist, s, nelems, i) {
1132
1133                 struct dma_debug_entry ref = {
1134                         .type           = dma_debug_sg,
1135                         .dev            = dev,
1136                         .paddr          = sg_phys(s),
1137                         .dev_addr       = sg_dma_address(s),
1138                         .size           = sg_dma_len(s),
1139                         .direction      = dir,
1140                         .sg_call_ents   = nelems,
1141                 };
1142
1143                 if (mapped_ents && i >= mapped_ents)
1144                         break;
1145
1146                 if (!i)
1147                         mapped_ents = get_nr_mapped_entries(dev, &ref);
1148
1149                 check_unmap(&ref);
1150         }
1151 }
1152 EXPORT_SYMBOL(debug_dma_unmap_sg);
1153
1154 void debug_dma_alloc_coherent(struct device *dev, size_t size,
1155                               dma_addr_t dma_addr, void *virt)
1156 {
1157         struct dma_debug_entry *entry;
1158
1159         if (unlikely(global_disable))
1160                 return;
1161
1162         if (unlikely(virt == NULL))
1163                 return;
1164
1165         entry = dma_entry_alloc();
1166         if (!entry)
1167                 return;
1168
1169         entry->type      = dma_debug_coherent;
1170         entry->dev       = dev;
1171         entry->paddr     = virt_to_phys(virt);
1172         entry->size      = size;
1173         entry->dev_addr  = dma_addr;
1174         entry->direction = DMA_BIDIRECTIONAL;
1175
1176         add_dma_entry(entry);
1177 }
1178 EXPORT_SYMBOL(debug_dma_alloc_coherent);
1179
1180 void debug_dma_free_coherent(struct device *dev, size_t size,
1181                          void *virt, dma_addr_t addr)
1182 {
1183         struct dma_debug_entry ref = {
1184                 .type           = dma_debug_coherent,
1185                 .dev            = dev,
1186                 .paddr          = virt_to_phys(virt),
1187                 .dev_addr       = addr,
1188                 .size           = size,
1189                 .direction      = DMA_BIDIRECTIONAL,
1190         };
1191
1192         if (unlikely(global_disable))
1193                 return;
1194
1195         check_unmap(&ref);
1196 }
1197 EXPORT_SYMBOL(debug_dma_free_coherent);
1198
1199 void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1200                                    size_t size, int direction)
1201 {
1202         struct dma_debug_entry ref;
1203
1204         if (unlikely(global_disable))
1205                 return;
1206
1207         ref.type         = dma_debug_single;
1208         ref.dev          = dev;
1209         ref.dev_addr     = dma_handle;
1210         ref.size         = size;
1211         ref.direction    = direction;
1212         ref.sg_call_ents = 0;
1213
1214         check_sync(dev, &ref, true);
1215 }
1216 EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
1217
1218 void debug_dma_sync_single_for_device(struct device *dev,
1219                                       dma_addr_t dma_handle, size_t size,
1220                                       int direction)
1221 {
1222         struct dma_debug_entry ref;
1223
1224         if (unlikely(global_disable))
1225                 return;
1226
1227         ref.type         = dma_debug_single;
1228         ref.dev          = dev;
1229         ref.dev_addr     = dma_handle;
1230         ref.size         = size;
1231         ref.direction    = direction;
1232         ref.sg_call_ents = 0;
1233
1234         check_sync(dev, &ref, false);
1235 }
1236 EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1237
1238 void debug_dma_sync_single_range_for_cpu(struct device *dev,
1239                                          dma_addr_t dma_handle,
1240                                          unsigned long offset, size_t size,
1241                                          int direction)
1242 {
1243         struct dma_debug_entry ref;
1244
1245         if (unlikely(global_disable))
1246                 return;
1247
1248         ref.type         = dma_debug_single;
1249         ref.dev          = dev;
1250         ref.dev_addr     = dma_handle;
1251         ref.size         = offset + size;
1252         ref.direction    = direction;
1253         ref.sg_call_ents = 0;
1254
1255         check_sync(dev, &ref, true);
1256 }
1257 EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1258
1259 void debug_dma_sync_single_range_for_device(struct device *dev,
1260                                             dma_addr_t dma_handle,
1261                                             unsigned long offset,
1262                                             size_t size, int direction)
1263 {
1264         struct dma_debug_entry ref;
1265
1266         if (unlikely(global_disable))
1267                 return;
1268
1269         ref.type         = dma_debug_single;
1270         ref.dev          = dev;
1271         ref.dev_addr     = dma_handle;
1272         ref.size         = offset + size;
1273         ref.direction    = direction;
1274         ref.sg_call_ents = 0;
1275
1276         check_sync(dev, &ref, false);
1277 }
1278 EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
1279
1280 void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1281                                int nelems, int direction)
1282 {
1283         struct scatterlist *s;
1284         int mapped_ents = 0, i;
1285
1286         if (unlikely(global_disable))
1287                 return;
1288
1289         for_each_sg(sg, s, nelems, i) {
1290
1291                 struct dma_debug_entry ref = {
1292                         .type           = dma_debug_sg,
1293                         .dev            = dev,
1294                         .paddr          = sg_phys(s),
1295                         .dev_addr       = sg_dma_address(s),
1296                         .size           = sg_dma_len(s),
1297                         .direction      = direction,
1298                         .sg_call_ents   = nelems,
1299                 };
1300
1301                 if (!i)
1302                         mapped_ents = get_nr_mapped_entries(dev, &ref);
1303
1304                 if (i >= mapped_ents)
1305                         break;
1306
1307                 check_sync(dev, &ref, true);
1308         }
1309 }
1310 EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
1311
1312 void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1313                                   int nelems, int direction)
1314 {
1315         struct scatterlist *s;
1316         int mapped_ents = 0, i;
1317
1318         if (unlikely(global_disable))
1319                 return;
1320
1321         for_each_sg(sg, s, nelems, i) {
1322
1323                 struct dma_debug_entry ref = {
1324                         .type           = dma_debug_sg,
1325                         .dev            = dev,
1326                         .paddr          = sg_phys(s),
1327                         .dev_addr       = sg_dma_address(s),
1328                         .size           = sg_dma_len(s),
1329                         .direction      = direction,
1330                         .sg_call_ents   = nelems,
1331                 };
1332                 if (!i)
1333                         mapped_ents = get_nr_mapped_entries(dev, &ref);
1334
1335                 if (i >= mapped_ents)
1336                         break;
1337
1338                 check_sync(dev, &ref, false);
1339         }
1340 }
1341 EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
1342
1343 static int __init dma_debug_driver_setup(char *str)
1344 {
1345         int i;
1346
1347         for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1348                 current_driver_name[i] = *str;
1349                 if (*str == 0)
1350                         break;
1351         }
1352
1353         if (current_driver_name[0])
1354                 pr_info("DMA-API: enable driver filter for driver [%s]\n",
1355                         current_driver_name);
1356
1357
1358         return 1;
1359 }
1360 __setup("dma_debug_driver=", dma_debug_driver_setup);