ARM: tegra: dvfs: Add interface to set fmax at vmin
[linux-3.10.git] / lib / dma-debug.c
1 /*
2  * Copyright (C) 2008 Advanced Micro Devices, Inc.
3  *
4  * Author: Joerg Roedel <joerg.roedel@amd.com>
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License version 2 as published
8  * by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
18  */
19
20 #include <linux/scatterlist.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/stacktrace.h>
23 #include <linux/dma-debug.h>
24 #include <linux/spinlock.h>
25 #include <linux/debugfs.h>
26 #include <linux/uaccess.h>
27 #include <linux/export.h>
28 #include <linux/device.h>
29 #include <linux/types.h>
30 #include <linux/sched.h>
31 #include <linux/ctype.h>
32 #include <linux/list.h>
33 #include <linux/slab.h>
34
35 #include <asm/sections.h>
36
37 #define HASH_SIZE       1024ULL
38 #define HASH_FN_SHIFT   13
39 #define HASH_FN_MASK    (HASH_SIZE - 1)
40
41 enum {
42         dma_debug_single,
43         dma_debug_page,
44         dma_debug_sg,
45         dma_debug_coherent,
46 };
47
48 enum map_err_types {
49         MAP_ERR_CHECK_NOT_APPLICABLE,
50         MAP_ERR_NOT_CHECKED,
51         MAP_ERR_CHECKED,
52 };
53
54 #define DMA_DEBUG_STACKTRACE_ENTRIES 5
55
56 struct dma_debug_entry {
57         struct list_head list;
58         struct device    *dev;
59         int              type;
60         phys_addr_t      paddr;
61         u64              dev_addr;
62         u64              size;
63         int              direction;
64         int              sg_call_ents;
65         int              sg_mapped_ents;
66         enum map_err_types  map_err_type;
67 #ifdef CONFIG_STACKTRACE
68         struct           stack_trace stacktrace;
69         unsigned long    st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
70 #endif
71 };
72
73 typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
74
75 struct hash_bucket {
76         struct list_head list;
77         spinlock_t lock;
78 } ____cacheline_aligned_in_smp;
79
80 /* Hash list to save the allocated dma addresses */
81 static struct hash_bucket dma_entry_hash[HASH_SIZE];
82 /* List of pre-allocated dma_debug_entry's */
83 static LIST_HEAD(free_entries);
84 /* Lock for the list above */
85 static DEFINE_SPINLOCK(free_entries_lock);
86
87 /* Global disable flag - will be set in case of an error */
88 static u32 global_disable __read_mostly;
89 /*
90  * The global_disable flag can be set in kernel command line, and
91  * dma_debug_init() is called much later than that. Mappings can happen before
92  * the init function is called, and thus no memory has been reserved for the
93  * entries. This out-of-memory situation flips enables the global_disable flag,
94  * which cannot then be enabled again. Use another flag for skipping the
95  * tracking if init hasn't been done yet.
96  */
97 static bool initialized __read_mostly;
98
99 /* Global error count */
100 static u32 error_count;
101
102 /* Global error show enable*/
103 static u32 show_all_errors __read_mostly;
104 /* Number of errors to show */
105 static u32 show_num_errors = 1;
106
107 static u32 num_free_entries;
108 static u32 min_free_entries;
109 static u32 nr_total_entries;
110
111 /* number of preallocated entries requested by kernel cmdline */
112 static u32 req_entries;
113
114 /* debugfs dentry's for the stuff above */
115 static struct dentry *dma_debug_dent        __read_mostly;
116 static struct dentry *global_disable_dent   __read_mostly;
117 static struct dentry *error_count_dent      __read_mostly;
118 static struct dentry *show_all_errors_dent  __read_mostly;
119 static struct dentry *show_num_errors_dent  __read_mostly;
120 static struct dentry *num_free_entries_dent __read_mostly;
121 static struct dentry *min_free_entries_dent __read_mostly;
122 static struct dentry *filter_dent           __read_mostly;
123
124 /* per-driver filter related state */
125
126 #define NAME_MAX_LEN    64
127
128 static char                  current_driver_name[NAME_MAX_LEN] __read_mostly;
129 static struct device_driver *current_driver                    __read_mostly;
130
131 static DEFINE_RWLOCK(driver_name_lock);
132
133 static const char *const maperr2str[] = {
134         [MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable",
135         [MAP_ERR_NOT_CHECKED] = "dma map error not checked",
136         [MAP_ERR_CHECKED] = "dma map error checked",
137 };
138
139 static const char *type2name[4] = { "single", "page",
140                                     "scather-gather", "coherent" };
141
142 static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
143                                    "DMA_FROM_DEVICE", "DMA_NONE" };
144
145 /* dma statistics per device */
146 struct dma_dev_info {
147         struct list_head list;
148         struct device *dev;
149         spinlock_t lock;        /* Protects dma_dev_info itself */
150
151         int current_allocs;
152         int total_allocs;
153         int max_allocs;
154
155         int current_alloc_size;
156         int total_alloc_size;
157         int max_alloc_size;
158 };
159
160 static LIST_HEAD(dev_info_list);
161 static DEFINE_SPINLOCK(dev_info_lock); /* Protects dev_info_list */
162
163 /*
164  * The access to some variables in this macro is racy. We can't use atomic_t
165  * here because all these variables are exported to debugfs. Some of them even
166  * writeable. This is also the reason why a lock won't help much. But anyway,
167  * the races are no big deal. Here is why:
168  *
169  *   error_count: the addition is racy, but the worst thing that can happen is
170  *                that we don't count some errors
171  *   show_num_errors: the subtraction is racy. Also no big deal because in
172  *                    worst case this will result in one warning more in the
173  *                    system log than the user configured. This variable is
174  *                    writeable via debugfs.
175  */
176 static inline void dump_entry_trace(struct dma_debug_entry *entry)
177 {
178 #ifdef CONFIG_STACKTRACE
179         if (entry) {
180                 pr_warning("Mapped at:\n");
181                 print_stack_trace(&entry->stacktrace, 0);
182         }
183 #endif
184 }
185
186 static bool driver_filter(struct device *dev)
187 {
188         struct device_driver *drv;
189         unsigned long flags;
190         bool ret;
191
192         /* driver filter off */
193         if (likely(!current_driver_name[0]))
194                 return true;
195
196         /* driver filter on and initialized */
197         if (current_driver && dev && dev->driver == current_driver)
198                 return true;
199
200         /* driver filter on, but we can't filter on a NULL device... */
201         if (!dev)
202                 return false;
203
204         if (current_driver || !current_driver_name[0])
205                 return false;
206
207         /* driver filter on but not yet initialized */
208         drv = dev->driver;
209         if (!drv)
210                 return false;
211
212         /* lock to protect against change of current_driver_name */
213         read_lock_irqsave(&driver_name_lock, flags);
214
215         ret = false;
216         if (drv->name &&
217             strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
218                 current_driver = drv;
219                 ret = true;
220         }
221
222         read_unlock_irqrestore(&driver_name_lock, flags);
223
224         return ret;
225 }
226
227 #define err_printk(dev, entry, format, arg...) do {                     \
228                 error_count += 1;                                       \
229                 if (driver_filter(dev) &&                               \
230                     (show_all_errors || show_num_errors > 0)) {         \
231                         WARN(1, "%s %s: " format,                       \
232                              dev ? dev_driver_string(dev) : "NULL",     \
233                              dev ? dev_name(dev) : "NULL", ## arg);     \
234                         dump_entry_trace(entry);                        \
235                 }                                                       \
236                 if (!show_all_errors && show_num_errors > 0)            \
237                         show_num_errors -= 1;                           \
238         } while (0);
239
240 /*
241  * Hash related functions
242  *
243  * Every DMA-API request is saved into a struct dma_debug_entry. To
244  * have quick access to these structs they are stored into a hash.
245  */
246 static int hash_fn(struct dma_debug_entry *entry)
247 {
248         /*
249          * Hash function is based on the dma address.
250          * We use bits 20-27 here as the index into the hash
251          */
252         return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
253 }
254
255 /*
256  * Request exclusive access to a hash bucket for a given dma_debug_entry.
257  */
258 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
259                                            unsigned long *flags)
260 {
261         int idx = hash_fn(entry);
262         unsigned long __flags;
263
264         spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
265         *flags = __flags;
266         return &dma_entry_hash[idx];
267 }
268
269 /*
270  * Give up exclusive access to the hash bucket
271  */
272 static void put_hash_bucket(struct hash_bucket *bucket,
273                             unsigned long *flags)
274 {
275         unsigned long __flags = *flags;
276
277         spin_unlock_irqrestore(&bucket->lock, __flags);
278 }
279
280 static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
281 {
282         return ((a->dev_addr == b->dev_addr) &&
283                 (a->dev == b->dev)) ? true : false;
284 }
285
286 static bool containing_match(struct dma_debug_entry *a,
287                              struct dma_debug_entry *b)
288 {
289         if (a->dev != b->dev)
290                 return false;
291
292         if ((b->dev_addr <= a->dev_addr) &&
293             ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))
294                 return true;
295
296         return false;
297 }
298
299 /*
300  * Search a given entry in the hash bucket list
301  */
302 static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,
303                                                   struct dma_debug_entry *ref,
304                                                   match_fn match)
305 {
306         struct dma_debug_entry *entry, *ret = NULL;
307         int matches = 0, match_lvl, last_lvl = -1;
308
309         list_for_each_entry(entry, &bucket->list, list) {
310                 if (!match(ref, entry))
311                         continue;
312
313                 /*
314                  * Some drivers map the same physical address multiple
315                  * times. Without a hardware IOMMU this results in the
316                  * same device addresses being put into the dma-debug
317                  * hash multiple times too. This can result in false
318                  * positives being reported. Therefore we implement a
319                  * best-fit algorithm here which returns the entry from
320                  * the hash which fits best to the reference value
321                  * instead of the first-fit.
322                  */
323                 matches += 1;
324                 match_lvl = 0;
325                 entry->size         == ref->size         ? ++match_lvl : 0;
326                 entry->type         == ref->type         ? ++match_lvl : 0;
327                 entry->direction    == ref->direction    ? ++match_lvl : 0;
328                 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
329
330                 if (match_lvl == 4) {
331                         /* perfect-fit - return the result */
332                         return entry;
333                 } else if (match_lvl > last_lvl) {
334                         /*
335                          * We found an entry that fits better then the
336                          * previous one or it is the 1st match.
337                          */
338                         last_lvl = match_lvl;
339                         ret      = entry;
340                 }
341         }
342
343         /*
344          * If we have multiple matches but no perfect-fit, just return
345          * NULL.
346          */
347         ret = (matches == 1) ? ret : NULL;
348
349         return ret;
350 }
351
352 static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,
353                                                  struct dma_debug_entry *ref)
354 {
355         return __hash_bucket_find(bucket, ref, exact_match);
356 }
357
358 static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,
359                                                    struct dma_debug_entry *ref,
360                                                    unsigned long *flags)
361 {
362
363         unsigned int max_range = dma_get_max_seg_size(ref->dev);
364         struct dma_debug_entry *entry, index = *ref;
365         unsigned int range = 0;
366
367         while (range <= max_range) {
368                 entry = __hash_bucket_find(*bucket, &index, containing_match);
369
370                 if (entry)
371                         return entry;
372
373                 /*
374                  * Nothing found, go back a hash bucket
375                  */
376                 put_hash_bucket(*bucket, flags);
377                 range          += (1 << HASH_FN_SHIFT);
378                 index.dev_addr -= (1 << HASH_FN_SHIFT);
379                 *bucket = get_hash_bucket(&index, flags);
380         }
381
382         return NULL;
383 }
384
385 /*
386  * Add an entry to a hash bucket
387  */
388 static void hash_bucket_add(struct hash_bucket *bucket,
389                             struct dma_debug_entry *entry)
390 {
391         list_add_tail(&entry->list, &bucket->list);
392 }
393
394 /*
395  * Remove entry from a hash bucket list
396  */
397 static void hash_bucket_del(struct dma_debug_entry *entry)
398 {
399         list_del(&entry->list);
400 }
401
402 /*
403  * Dump mapping entries for debugging purposes
404  */
405 void debug_dma_dump_mappings(struct device *dev)
406 {
407         int idx;
408
409         for (idx = 0; idx < HASH_SIZE; idx++) {
410                 struct hash_bucket *bucket = &dma_entry_hash[idx];
411                 struct dma_debug_entry *entry;
412                 unsigned long flags;
413
414                 spin_lock_irqsave(&bucket->lock, flags);
415
416                 list_for_each_entry(entry, &bucket->list, list) {
417                         if (!dev || dev == entry->dev) {
418                                 dev_info(entry->dev,
419                                          "%s idx %d P=%Lx D=%Lx L=%Lx %s %s\n",
420                                          type2name[entry->type], idx,
421                                          (unsigned long long)entry->paddr,
422                                          entry->dev_addr, entry->size,
423                                          dir2name[entry->direction],
424                                          maperr2str[entry->map_err_type]);
425                         }
426                 }
427
428                 spin_unlock_irqrestore(&bucket->lock, flags);
429         }
430 }
431 EXPORT_SYMBOL(debug_dma_dump_mappings);
432
433 /*
434  * device info snapshot updating functions
435  */
436 static void ____dev_info_incr(struct dma_dev_info *info,
437                               struct dma_debug_entry *entry)
438 {
439         unsigned long flags;
440
441         spin_lock_irqsave(&info->lock, flags);
442
443         info->current_allocs++;
444         info->total_allocs++;
445         if (info->current_allocs > info->max_allocs)
446                 info->max_allocs = info->current_allocs;
447
448         info->current_alloc_size += entry->size;
449         info->total_alloc_size += entry->size;
450         if (info->current_alloc_size > info->max_alloc_size)
451                 info->max_alloc_size = info->current_alloc_size;
452
453         spin_unlock_irqrestore(&info->lock, flags);
454 }
455
456 static void ____dev_info_decr(struct dma_dev_info *info,
457                               struct dma_debug_entry *entry)
458 {
459         unsigned long flags;
460
461         spin_lock_irqsave(&info->lock, flags);
462
463         info->current_allocs--;
464         info->current_alloc_size -= entry->size;
465
466         spin_unlock_irqrestore(&info->lock, flags);
467 }
468
469 static void __dev_info_fn(struct dma_debug_entry *entry,
470          void (*fn)(struct dma_dev_info *, struct dma_debug_entry *))
471 {
472         struct dma_dev_info *info;
473         unsigned long flags;
474
475         spin_lock_irqsave(&dev_info_lock, flags);
476
477         list_for_each_entry(info, &dev_info_list, list)
478                 if (info->dev == entry->dev)
479                         goto found;
480
481         info = kzalloc(sizeof(*info), GFP_KERNEL);
482         if (!info) {
483                 dev_err(entry->dev, "Out of memory at %s\n", __func__);
484                 spin_unlock_irqrestore(&dev_info_lock, flags);
485                 return;
486         }
487
488         spin_lock_init(&info->lock);
489         info->dev = entry->dev;
490         list_add(&info->list, &dev_info_list);
491 found:
492         spin_unlock_irqrestore(&dev_info_lock, flags);
493         fn(info, entry);
494 }
495
496 static inline void dev_info_alloc(struct dma_debug_entry *entry)
497 {
498         __dev_info_fn(entry, ____dev_info_incr);
499 }
500
501 static inline void dev_info_free(struct dma_debug_entry *entry)
502 {
503         __dev_info_fn(entry, ____dev_info_decr);
504 }
505
506 /*
507  * Wrapper function for adding an entry to the hash.
508  * This function takes care of locking itself.
509  */
510 static void add_dma_entry(struct dma_debug_entry *entry)
511 {
512         struct hash_bucket *bucket;
513         unsigned long flags;
514
515         bucket = get_hash_bucket(entry, &flags);
516         hash_bucket_add(bucket, entry);
517         put_hash_bucket(bucket, &flags);
518
519         dev_info_alloc(entry);
520 }
521
522 static struct dma_debug_entry *__dma_entry_alloc(void)
523 {
524         struct dma_debug_entry *entry;
525
526         entry = list_entry(free_entries.next, struct dma_debug_entry, list);
527         list_del(&entry->list);
528         memset(entry, 0, sizeof(*entry));
529
530         num_free_entries -= 1;
531         if (num_free_entries < min_free_entries)
532                 min_free_entries = num_free_entries;
533
534         return entry;
535 }
536
537 /* struct dma_entry allocator
538  *
539  * The next two functions implement the allocator for
540  * struct dma_debug_entries.
541  */
542 static struct dma_debug_entry *dma_entry_alloc(void)
543 {
544         struct dma_debug_entry *entry;
545         unsigned long flags;
546
547         spin_lock_irqsave(&free_entries_lock, flags);
548
549         if (list_empty(&free_entries)) {
550                 pr_err("DMA-API: debugging out of memory - disabling\n");
551                 global_disable = true;
552                 spin_unlock_irqrestore(&free_entries_lock, flags);
553                 return NULL;
554         }
555
556         entry = __dma_entry_alloc();
557
558         spin_unlock_irqrestore(&free_entries_lock, flags);
559
560 #ifdef CONFIG_STACKTRACE
561         entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
562         entry->stacktrace.entries = entry->st_entries;
563         entry->stacktrace.skip = 2;
564         save_stack_trace(&entry->stacktrace);
565 #endif
566
567         return entry;
568 }
569
570 static void dma_entry_free(struct dma_debug_entry *entry)
571 {
572         unsigned long flags;
573
574         /*
575          * add to beginning of the list - this way the entries are
576          * more likely cache hot when they are reallocated.
577          */
578         spin_lock_irqsave(&free_entries_lock, flags);
579         list_add(&entry->list, &free_entries);
580         num_free_entries += 1;
581         spin_unlock_irqrestore(&free_entries_lock, flags);
582 }
583
584 int dma_debug_resize_entries(u32 num_entries)
585 {
586         int i, delta, ret = 0;
587         unsigned long flags;
588         struct dma_debug_entry *entry;
589         LIST_HEAD(tmp);
590
591         spin_lock_irqsave(&free_entries_lock, flags);
592
593         if (nr_total_entries < num_entries) {
594                 delta = num_entries - nr_total_entries;
595
596                 spin_unlock_irqrestore(&free_entries_lock, flags);
597
598                 for (i = 0; i < delta; i++) {
599                         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
600                         if (!entry)
601                                 break;
602
603                         list_add_tail(&entry->list, &tmp);
604                 }
605
606                 spin_lock_irqsave(&free_entries_lock, flags);
607
608                 list_splice(&tmp, &free_entries);
609                 nr_total_entries += i;
610                 num_free_entries += i;
611         } else {
612                 delta = nr_total_entries - num_entries;
613
614                 for (i = 0; i < delta && !list_empty(&free_entries); i++) {
615                         entry = __dma_entry_alloc();
616                         kfree(entry);
617                 }
618
619                 nr_total_entries -= i;
620         }
621
622         if (nr_total_entries != num_entries)
623                 ret = 1;
624
625         spin_unlock_irqrestore(&free_entries_lock, flags);
626
627         return ret;
628 }
629 EXPORT_SYMBOL(dma_debug_resize_entries);
630
631 /*
632  * DMA-API debugging init code
633  *
634  * The init code does two things:
635  *   1. Initialize core data structures
636  *   2. Preallocate a given number of dma_debug_entry structs
637  */
638
639 static int prealloc_memory(u32 num_entries)
640 {
641         struct dma_debug_entry *entry, *next_entry;
642         int i;
643
644         for (i = 0; i < num_entries; ++i) {
645                 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
646                 if (!entry)
647                         goto out_err;
648
649                 list_add_tail(&entry->list, &free_entries);
650         }
651
652         num_free_entries = num_entries;
653         min_free_entries = num_entries;
654
655         pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
656
657         return 0;
658
659 out_err:
660
661         list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
662                 list_del(&entry->list);
663                 kfree(entry);
664         }
665
666         return -ENOMEM;
667 }
668
669 static ssize_t filter_read(struct file *file, char __user *user_buf,
670                            size_t count, loff_t *ppos)
671 {
672         char buf[NAME_MAX_LEN + 1];
673         unsigned long flags;
674         int len;
675
676         if (!current_driver_name[0])
677                 return 0;
678
679         /*
680          * We can't copy to userspace directly because current_driver_name can
681          * only be read under the driver_name_lock with irqs disabled. So
682          * create a temporary copy first.
683          */
684         read_lock_irqsave(&driver_name_lock, flags);
685         len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
686         read_unlock_irqrestore(&driver_name_lock, flags);
687
688         return simple_read_from_buffer(user_buf, count, ppos, buf, len);
689 }
690
691 static ssize_t filter_write(struct file *file, const char __user *userbuf,
692                             size_t count, loff_t *ppos)
693 {
694         char buf[NAME_MAX_LEN];
695         unsigned long flags;
696         size_t len;
697         int i;
698
699         /*
700          * We can't copy from userspace directly. Access to
701          * current_driver_name is protected with a write_lock with irqs
702          * disabled. Since copy_from_user can fault and may sleep we
703          * need to copy to temporary buffer first
704          */
705         len = min(count, (size_t)(NAME_MAX_LEN - 1));
706         if (copy_from_user(buf, userbuf, len))
707                 return -EFAULT;
708
709         buf[len] = 0;
710
711         write_lock_irqsave(&driver_name_lock, flags);
712
713         /*
714          * Now handle the string we got from userspace very carefully.
715          * The rules are:
716          *         - only use the first token we got
717          *         - token delimiter is everything looking like a space
718          *           character (' ', '\n', '\t' ...)
719          *
720          */
721         if (!isalnum(buf[0])) {
722                 /*
723                  * If the first character userspace gave us is not
724                  * alphanumerical then assume the filter should be
725                  * switched off.
726                  */
727                 if (current_driver_name[0])
728                         pr_info("DMA-API: switching off dma-debug driver filter\n");
729                 current_driver_name[0] = 0;
730                 current_driver = NULL;
731                 goto out_unlock;
732         }
733
734         /*
735          * Now parse out the first token and use it as the name for the
736          * driver to filter for.
737          */
738         for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
739                 current_driver_name[i] = buf[i];
740                 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
741                         break;
742         }
743         current_driver_name[i] = 0;
744         current_driver = NULL;
745
746         pr_info("DMA-API: enable driver filter for driver [%s]\n",
747                 current_driver_name);
748
749 out_unlock:
750         write_unlock_irqrestore(&driver_name_lock, flags);
751
752         return count;
753 }
754
755 static inline void seq_print_ip_sym(struct seq_file *s, unsigned long ip)
756 {
757         seq_printf(s, "[<%p>] %pS\n", (void *)ip, (void *)ip);
758 }
759
760 void seq_print_trace(struct seq_file *s, struct stack_trace *trace)
761 {
762         int i;
763
764         if (WARN_ON(!trace->entries))
765                 return;
766
767         for (i = trace->skip; i < trace->nr_entries; i++)
768                 seq_print_ip_sym(s, trace->entries[i]);
769 }
770
771 /*
772  * Print all map entries just in the order they are stored. We assume that the
773  * user will be able to parse this later anyway. Detailed output includes stack
774  * traces of allocations.
775  */
776 void seq_print_dma_mappings(struct seq_file *s, int detail)
777 {
778         int idx;
779
780         for (idx = 0; idx < HASH_SIZE; idx++) {
781                 struct hash_bucket *bucket = &dma_entry_hash[idx];
782                 struct dma_debug_entry *entry;
783                 unsigned long flags;
784
785                 spin_lock_irqsave(&bucket->lock, flags);
786
787                 list_for_each_entry(entry, &bucket->list, list) {
788                         seq_printf(s,
789                                    "    %s %s idx %d P=%llx D=%llx L=%llx %s A=%s\n",
790                                    dev_name(entry->dev),
791                                    type2name[entry->type], idx,
792                                    (u64)entry->paddr,
793                                    entry->dev_addr, entry->size,
794                                    dir2name[entry->direction],
795                                    debug_dma_platformdata(entry->dev));
796
797                         if (detail)
798                                 seq_print_trace(s, &entry->stacktrace);
799                 }
800
801                 spin_unlock_irqrestore(&bucket->lock, flags);
802         }
803 }
804
805 void __weak dma_debugfs_platform_info(struct dentry *dent)
806 {
807 }
808
809 static int _dump_allocs(struct seq_file *s, void *data)
810 {
811         int detail = (int)s->private;
812
813         seq_print_dma_mappings(s, detail);
814         return 0;
815 }
816
817 static int _dump_dev_info(struct seq_file *s, void *data)
818 {
819         struct dma_dev_info *i;
820         unsigned long flags;
821
822         spin_lock_irqsave(&dev_info_lock, flags);
823
824         list_for_each_entry(i, &dev_info_list, list)
825                 seq_printf(s,
826                            "dev=%s curallocs=%d totallocs=%d maxallocs=%d cursize=%d totsize=%d maxsize=%d\n",
827                            dev_name(i->dev), i->current_allocs, i->total_allocs,
828                            i->max_allocs, i->current_alloc_size,
829                            i->total_alloc_size, i->max_alloc_size);
830
831         spin_unlock_irqrestore(&dev_info_lock, flags);
832         return 0;
833 }
834
835 #define DEFINE_DEBUGFS(__name, __func, __data)                          \
836 static int __name ## _open(struct inode *inode, struct file *file)      \
837 {                                                                       \
838         return single_open(file, __func, __data);                       \
839 }                                                                       \
840 static const struct file_operations __name ## _fops = {                 \
841         .open           = __name ## _open,                              \
842         .read           = seq_read,                                     \
843         .llseek         = seq_lseek,                                    \
844         .release        = single_release,                               \
845 }
846
847 DEFINE_DEBUGFS(_dump_allocs, _dump_allocs, NULL);
848 DEFINE_DEBUGFS(_dump_allocs_detail, _dump_allocs, (void *)1);
849 DEFINE_DEBUGFS(_dump_dev_info, _dump_dev_info, NULL);
850 #undef DEFINE_DEBUGFS
851
852 static int map_dump_debug_fs_init(void)
853 {
854 #define CREATE_FILE(name) \
855         debugfs_create_file(#name, S_IRUGO, \
856                                 dma_debug_dent, NULL, \
857                                 &_##name##_fops)
858
859         if (!CREATE_FILE(dump_allocs))
860                 return -ENOMEM;
861
862         if (!CREATE_FILE(dump_allocs_detail))
863                 return -ENOMEM;
864
865         if (!CREATE_FILE(dump_dev_info))
866                 return -ENOMEM;
867
868 #undef CREATE_FILE
869
870         dma_debugfs_platform_info(dma_debug_dent);
871         return 0;
872 }
873
874 static const struct file_operations filter_fops = {
875         .read  = filter_read,
876         .write = filter_write,
877         .llseek = default_llseek,
878 };
879
880 static int dma_debug_fs_init(void)
881 {
882         dma_debug_dent = debugfs_create_dir("dma-api", NULL);
883         if (!dma_debug_dent) {
884                 pr_err("DMA-API: can not create debugfs directory\n");
885                 return -ENOMEM;
886         }
887
888         global_disable_dent = debugfs_create_bool("disabled", 0444,
889                         dma_debug_dent,
890                         &global_disable);
891         if (!global_disable_dent)
892                 goto out_err;
893
894         error_count_dent = debugfs_create_u32("error_count", 0444,
895                         dma_debug_dent, &error_count);
896         if (!error_count_dent)
897                 goto out_err;
898
899         show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
900                         dma_debug_dent,
901                         &show_all_errors);
902         if (!show_all_errors_dent)
903                 goto out_err;
904
905         show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
906                         dma_debug_dent,
907                         &show_num_errors);
908         if (!show_num_errors_dent)
909                 goto out_err;
910
911         num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
912                         dma_debug_dent,
913                         &num_free_entries);
914         if (!num_free_entries_dent)
915                 goto out_err;
916
917         min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
918                         dma_debug_dent,
919                         &min_free_entries);
920         if (!min_free_entries_dent)
921                 goto out_err;
922
923         filter_dent = debugfs_create_file("driver_filter", 0644,
924                                           dma_debug_dent, NULL, &filter_fops);
925         if (!filter_dent)
926                 goto out_err;
927
928         if (map_dump_debug_fs_init())
929                 goto out_err;
930
931         return 0;
932
933 out_err:
934         debugfs_remove_recursive(dma_debug_dent);
935
936         return -ENOMEM;
937 }
938
939 static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
940 {
941         struct dma_debug_entry *entry;
942         unsigned long flags;
943         int count = 0, i;
944
945         local_irq_save(flags);
946
947         for (i = 0; i < HASH_SIZE; ++i) {
948                 spin_lock(&dma_entry_hash[i].lock);
949                 list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
950                         if (entry->dev == dev) {
951                                 count += 1;
952                                 *out_entry = entry;
953                         }
954                 }
955                 spin_unlock(&dma_entry_hash[i].lock);
956         }
957
958         local_irq_restore(flags);
959
960         return count;
961 }
962
963 static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
964 {
965         struct device *dev = data;
966         struct dma_debug_entry *uninitialized_var(entry);
967         int count;
968
969         if (!initialized || global_disable)
970                 return 0;
971
972         switch (action) {
973         case BUS_NOTIFY_UNBOUND_DRIVER:
974                 count = device_dma_allocations(dev, &entry);
975                 if (count == 0)
976                         break;
977                 err_printk(dev, entry, "DMA-API: device driver has pending "
978                                 "DMA allocations while released from device "
979                                 "[count=%d]\n"
980                                 "One of leaked entries details: "
981                                 "[device address=0x%016llx] [size=%llu bytes] "
982                                 "[mapped with %s] [mapped as %s]\n",
983                         count, entry->dev_addr, entry->size,
984                         dir2name[entry->direction], type2name[entry->type]);
985                 break;
986         default:
987                 break;
988         }
989
990         return 0;
991 }
992
993 void dma_debug_add_bus(struct bus_type *bus)
994 {
995         struct notifier_block *nb;
996
997         if (!initialized || global_disable)
998                 return;
999
1000         nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
1001         if (nb == NULL) {
1002                 pr_err("dma_debug_add_bus: out of memory\n");
1003                 return;
1004         }
1005
1006         nb->notifier_call = dma_debug_device_change;
1007
1008         bus_register_notifier(bus, nb);
1009 }
1010
1011 /*
1012  * Let the architectures decide how many entries should be preallocated.
1013  */
1014 void dma_debug_init(u32 num_entries)
1015 {
1016         int i;
1017
1018         if (global_disable)
1019                 return;
1020
1021         for (i = 0; i < HASH_SIZE; ++i) {
1022                 INIT_LIST_HEAD(&dma_entry_hash[i].list);
1023                 spin_lock_init(&dma_entry_hash[i].lock);
1024         }
1025
1026         if (dma_debug_fs_init() != 0) {
1027                 pr_err("DMA-API: error creating debugfs entries - disabling\n");
1028                 global_disable = true;
1029
1030                 return;
1031         }
1032
1033         if (req_entries)
1034                 num_entries = req_entries;
1035
1036         if (prealloc_memory(num_entries) != 0) {
1037                 pr_err("DMA-API: debugging out of memory error - disabled\n");
1038                 global_disable = true;
1039
1040                 return;
1041         }
1042
1043         nr_total_entries = num_free_entries;
1044
1045         pr_info("DMA-API: debugging enabled by kernel config\n");
1046         initialized = true;
1047 }
1048
1049 static __init int dma_debug_cmdline(char *str)
1050 {
1051         if (!str)
1052                 return -EINVAL;
1053
1054         if (strncmp(str, "off", 3) == 0) {
1055                 pr_info("DMA-API: debugging disabled on kernel command line\n");
1056                 global_disable = true;
1057         }
1058
1059         return 0;
1060 }
1061
1062 static __init int dma_debug_entries_cmdline(char *str)
1063 {
1064         int res;
1065
1066         if (!str)
1067                 return -EINVAL;
1068
1069         res = get_option(&str, &req_entries);
1070
1071         if (!res)
1072                 req_entries = 0;
1073
1074         return 0;
1075 }
1076
1077 __setup("dma_debug=", dma_debug_cmdline);
1078 __setup("dma_debug_entries=", dma_debug_entries_cmdline);
1079
1080 static void check_unmap(struct dma_debug_entry *ref)
1081 {
1082         struct dma_debug_entry *entry;
1083         struct hash_bucket *bucket;
1084         unsigned long flags;
1085
1086         bucket = get_hash_bucket(ref, &flags);
1087         entry = bucket_find_exact(bucket, ref);
1088
1089         if (!entry) {
1090                 /* must drop lock before calling dma_mapping_error */
1091                 put_hash_bucket(bucket, &flags);
1092
1093                 if (dma_mapping_error(ref->dev, ref->dev_addr)) {
1094                         err_printk(ref->dev, NULL,
1095                                    "DMA-API: device driver tries to free an "
1096                                    "invalid DMA memory address\n");
1097                 } else {
1098                         err_printk(ref->dev, NULL,
1099                                    "DMA-API: device driver tries to free DMA "
1100                                    "memory it has not allocated [device "
1101                                    "address=0x%016llx] [size=%llu bytes]\n",
1102                                    ref->dev_addr, ref->size);
1103                 }
1104                 return;
1105         }
1106
1107         if (ref->size != entry->size) {
1108                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1109                            "DMA memory with different size "
1110                            "[device address=0x%016llx] [map size=%llu bytes] "
1111                            "[unmap size=%llu bytes]\n",
1112                            ref->dev_addr, entry->size, ref->size);
1113         }
1114
1115         if (ref->type != entry->type) {
1116                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1117                            "DMA memory with wrong function "
1118                            "[device address=0x%016llx] [size=%llu bytes] "
1119                            "[mapped as %s] [unmapped as %s]\n",
1120                            ref->dev_addr, ref->size,
1121                            type2name[entry->type], type2name[ref->type]);
1122         } else if ((entry->type == dma_debug_coherent) &&
1123                    (ref->paddr != entry->paddr)) {
1124                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1125                            "DMA memory with different CPU address "
1126                            "[device address=0x%016llx] [size=%llu bytes] "
1127                            "[cpu alloc address=0x%016llx] "
1128                            "[cpu free address=0x%016llx]",
1129                            ref->dev_addr, ref->size,
1130                            (unsigned long long)entry->paddr,
1131                            (unsigned long long)ref->paddr);
1132         }
1133
1134         if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1135             ref->sg_call_ents != entry->sg_call_ents) {
1136                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1137                            "DMA sg list with different entry count "
1138                            "[map count=%d] [unmap count=%d]\n",
1139                            entry->sg_call_ents, ref->sg_call_ents);
1140         }
1141
1142         /*
1143          * This may be no bug in reality - but most implementations of the
1144          * DMA API don't handle this properly, so check for it here
1145          */
1146         if (ref->direction != entry->direction) {
1147                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1148                            "DMA memory with different direction "
1149                            "[device address=0x%016llx] [size=%llu bytes] "
1150                            "[mapped with %s] [unmapped with %s]\n",
1151                            ref->dev_addr, ref->size,
1152                            dir2name[entry->direction],
1153                            dir2name[ref->direction]);
1154         }
1155
1156         if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1157                 err_printk(ref->dev, entry,
1158                            "DMA-API: device driver failed to check map error"
1159                            "[device address=0x%016llx] [size=%llu bytes] "
1160                            "[mapped as %s]",
1161                            ref->dev_addr, ref->size,
1162                            type2name[entry->type]);
1163         }
1164
1165         dev_info_free(entry);
1166
1167         hash_bucket_del(entry);
1168         dma_entry_free(entry);
1169
1170         put_hash_bucket(bucket, &flags);
1171 }
1172
1173 static void check_for_stack(struct device *dev, void *addr)
1174 {
1175         if (object_is_on_stack(addr))
1176                 err_printk(dev, NULL, "DMA-API: device driver maps memory from"
1177                                 "stack [addr=%p]\n", addr);
1178 }
1179
1180 static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
1181 {
1182         unsigned long a1 = (unsigned long)addr;
1183         unsigned long b1 = a1 + len;
1184         unsigned long a2 = (unsigned long)start;
1185         unsigned long b2 = (unsigned long)end;
1186
1187         return !(b1 <= a2 || a1 >= b2);
1188 }
1189
1190 static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
1191 {
1192         if (overlap(addr, len, _text, _etext) ||
1193             overlap(addr, len, __start_rodata, __end_rodata))
1194                 err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
1195 }
1196
1197 static void check_sync(struct device *dev,
1198                        struct dma_debug_entry *ref,
1199                        bool to_cpu)
1200 {
1201         struct dma_debug_entry *entry;
1202         struct hash_bucket *bucket;
1203         unsigned long flags;
1204
1205         bucket = get_hash_bucket(ref, &flags);
1206
1207         entry = bucket_find_contain(&bucket, ref, &flags);
1208
1209         if (!entry) {
1210                 err_printk(dev, NULL, "DMA-API: device driver tries "
1211                                 "to sync DMA memory it has not allocated "
1212                                 "[device address=0x%016llx] [size=%llu bytes]\n",
1213                                 (unsigned long long)ref->dev_addr, ref->size);
1214                 goto out;
1215         }
1216
1217         if (ref->size > entry->size) {
1218                 err_printk(dev, entry, "DMA-API: device driver syncs"
1219                                 " DMA memory outside allocated range "
1220                                 "[device address=0x%016llx] "
1221                                 "[allocation size=%llu bytes] "
1222                                 "[sync offset+size=%llu]\n",
1223                                 entry->dev_addr, entry->size,
1224                                 ref->size);
1225         }
1226
1227         if (entry->direction == DMA_BIDIRECTIONAL)
1228                 goto out;
1229
1230         if (ref->direction != entry->direction) {
1231                 err_printk(dev, entry, "DMA-API: device driver syncs "
1232                                 "DMA memory with different direction "
1233                                 "[device address=0x%016llx] [size=%llu bytes] "
1234                                 "[mapped with %s] [synced with %s]\n",
1235                                 (unsigned long long)ref->dev_addr, entry->size,
1236                                 dir2name[entry->direction],
1237                                 dir2name[ref->direction]);
1238         }
1239
1240         if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
1241                       !(ref->direction == DMA_TO_DEVICE))
1242                 err_printk(dev, entry, "DMA-API: device driver syncs "
1243                                 "device read-only DMA memory for cpu "
1244                                 "[device address=0x%016llx] [size=%llu bytes] "
1245                                 "[mapped with %s] [synced with %s]\n",
1246                                 (unsigned long long)ref->dev_addr, entry->size,
1247                                 dir2name[entry->direction],
1248                                 dir2name[ref->direction]);
1249
1250         if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
1251                        !(ref->direction == DMA_FROM_DEVICE))
1252                 err_printk(dev, entry, "DMA-API: device driver syncs "
1253                                 "device write-only DMA memory to device "
1254                                 "[device address=0x%016llx] [size=%llu bytes] "
1255                                 "[mapped with %s] [synced with %s]\n",
1256                                 (unsigned long long)ref->dev_addr, entry->size,
1257                                 dir2name[entry->direction],
1258                                 dir2name[ref->direction]);
1259
1260 out:
1261         put_hash_bucket(bucket, &flags);
1262 }
1263
1264 void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
1265                         size_t size, int direction, dma_addr_t dma_addr,
1266                         bool map_single)
1267 {
1268         struct dma_debug_entry *entry;
1269
1270         if (unlikely(!initialized || global_disable))
1271                 return;
1272
1273         if (dma_mapping_error(dev, dma_addr))
1274                 return;
1275
1276         entry = dma_entry_alloc();
1277         if (!entry)
1278                 return;
1279
1280         entry->dev       = dev;
1281         entry->type      = dma_debug_page;
1282         entry->paddr     = page_to_phys(page) + offset;
1283         entry->dev_addr  = dma_addr;
1284         entry->size      = size;
1285         entry->direction = direction;
1286         entry->map_err_type = MAP_ERR_NOT_CHECKED;
1287
1288         if (map_single)
1289                 entry->type = dma_debug_single;
1290
1291         if (!PageHighMem(page)) {
1292                 void *addr = page_address(page) + offset;
1293
1294                 check_for_stack(dev, addr);
1295                 check_for_illegal_area(dev, addr, size);
1296         }
1297
1298         add_dma_entry(entry);
1299 }
1300 EXPORT_SYMBOL(debug_dma_map_page);
1301
1302 void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
1303 {
1304         struct dma_debug_entry ref;
1305         struct dma_debug_entry *entry;
1306         struct hash_bucket *bucket;
1307         unsigned long flags;
1308
1309         if (unlikely(!initialized || global_disable))
1310                 return;
1311
1312         ref.dev = dev;
1313         ref.dev_addr = dma_addr;
1314         bucket = get_hash_bucket(&ref, &flags);
1315
1316         list_for_each_entry(entry, &bucket->list, list) {
1317                 if (!exact_match(&ref, entry))
1318                         continue;
1319
1320                 /*
1321                  * The same physical address can be mapped multiple
1322                  * times. Without a hardware IOMMU this results in the
1323                  * same device addresses being put into the dma-debug
1324                  * hash multiple times too. This can result in false
1325                  * positives being reported. Therefore we implement a
1326                  * best-fit algorithm here which updates the first entry
1327                  * from the hash which fits the reference value and is
1328                  * not currently listed as being checked.
1329                  */
1330                 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1331                         entry->map_err_type = MAP_ERR_CHECKED;
1332                         break;
1333                 }
1334         }
1335
1336         put_hash_bucket(bucket, &flags);
1337 }
1338 EXPORT_SYMBOL(debug_dma_mapping_error);
1339
1340 void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1341                           size_t size, int direction, bool map_single)
1342 {
1343         struct dma_debug_entry ref = {
1344                 .type           = dma_debug_page,
1345                 .dev            = dev,
1346                 .dev_addr       = addr,
1347                 .size           = size,
1348                 .direction      = direction,
1349         };
1350
1351         if (unlikely(!initialized || global_disable))
1352                 return;
1353
1354         if (map_single)
1355                 ref.type = dma_debug_single;
1356
1357         check_unmap(&ref);
1358 }
1359 EXPORT_SYMBOL(debug_dma_unmap_page);
1360
1361 void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1362                       int nents, int mapped_ents, int direction)
1363 {
1364         struct dma_debug_entry *entry;
1365         struct scatterlist *s;
1366         int i;
1367
1368         if (unlikely(!initialized || global_disable))
1369                 return;
1370
1371         for_each_sg(sg, s, mapped_ents, i) {
1372                 entry = dma_entry_alloc();
1373                 if (!entry)
1374                         return;
1375
1376                 entry->type           = dma_debug_sg;
1377                 entry->dev            = dev;
1378                 entry->paddr          = sg_phys(s);
1379                 entry->size           = sg_dma_len(s);
1380                 entry->dev_addr       = sg_dma_address(s);
1381                 entry->direction      = direction;
1382                 entry->sg_call_ents   = nents;
1383                 entry->sg_mapped_ents = mapped_ents;
1384
1385                 if (!PageHighMem(sg_page(s))) {
1386                         check_for_stack(dev, sg_virt(s));
1387                         check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
1388                 }
1389
1390                 add_dma_entry(entry);
1391         }
1392 }
1393 EXPORT_SYMBOL(debug_dma_map_sg);
1394
1395 static int get_nr_mapped_entries(struct device *dev,
1396                                  struct dma_debug_entry *ref)
1397 {
1398         struct dma_debug_entry *entry;
1399         struct hash_bucket *bucket;
1400         unsigned long flags;
1401         int mapped_ents;
1402
1403         bucket       = get_hash_bucket(ref, &flags);
1404         entry        = bucket_find_exact(bucket, ref);
1405         mapped_ents  = 0;
1406
1407         if (entry)
1408                 mapped_ents = entry->sg_mapped_ents;
1409         put_hash_bucket(bucket, &flags);
1410
1411         return mapped_ents;
1412 }
1413
1414 void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1415                         int nelems, int dir)
1416 {
1417         struct scatterlist *s;
1418         int mapped_ents = 0, i;
1419
1420         if (unlikely(!initialized || global_disable))
1421                 return;
1422
1423         for_each_sg(sglist, s, nelems, i) {
1424
1425                 struct dma_debug_entry ref = {
1426                         .type           = dma_debug_sg,
1427                         .dev            = dev,
1428                         .paddr          = sg_phys(s),
1429                         .dev_addr       = sg_dma_address(s),
1430                         .size           = sg_dma_len(s),
1431                         .direction      = dir,
1432                         .sg_call_ents   = nelems,
1433                 };
1434
1435                 if (mapped_ents && i >= mapped_ents)
1436                         break;
1437
1438                 if (!i)
1439                         mapped_ents = get_nr_mapped_entries(dev, &ref);
1440
1441                 check_unmap(&ref);
1442         }
1443 }
1444 EXPORT_SYMBOL(debug_dma_unmap_sg);
1445
1446 void debug_dma_alloc_coherent(struct device *dev, size_t size,
1447                               dma_addr_t dma_addr, void *virt)
1448 {
1449         struct dma_debug_entry *entry;
1450
1451         if (unlikely(!initialized || global_disable))
1452                 return;
1453
1454         if (unlikely(virt == NULL))
1455                 return;
1456
1457         entry = dma_entry_alloc();
1458         if (!entry)
1459                 return;
1460
1461         entry->type      = dma_debug_coherent;
1462         entry->dev       = dev;
1463         entry->paddr     = virt_to_phys(virt);
1464         entry->size      = size;
1465         entry->dev_addr  = dma_addr;
1466         entry->direction = DMA_BIDIRECTIONAL;
1467
1468         add_dma_entry(entry);
1469 }
1470 EXPORT_SYMBOL(debug_dma_alloc_coherent);
1471
1472 void debug_dma_free_coherent(struct device *dev, size_t size,
1473                          void *virt, dma_addr_t addr)
1474 {
1475         struct dma_debug_entry ref = {
1476                 .type           = dma_debug_coherent,
1477                 .dev            = dev,
1478                 .paddr          = virt_to_phys(virt),
1479                 .dev_addr       = addr,
1480                 .size           = size,
1481                 .direction      = DMA_BIDIRECTIONAL,
1482         };
1483
1484         if (unlikely(!initialized || global_disable))
1485                 return;
1486
1487         check_unmap(&ref);
1488 }
1489 EXPORT_SYMBOL(debug_dma_free_coherent);
1490
1491 void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1492                                    size_t size, int direction)
1493 {
1494         struct dma_debug_entry ref;
1495
1496         if (unlikely(!initialized || global_disable))
1497                 return;
1498
1499         ref.type         = dma_debug_single;
1500         ref.dev          = dev;
1501         ref.dev_addr     = dma_handle;
1502         ref.size         = size;
1503         ref.direction    = direction;
1504         ref.sg_call_ents = 0;
1505
1506         check_sync(dev, &ref, true);
1507 }
1508 EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
1509
1510 void debug_dma_sync_single_for_device(struct device *dev,
1511                                       dma_addr_t dma_handle, size_t size,
1512                                       int direction)
1513 {
1514         struct dma_debug_entry ref;
1515
1516         if (unlikely(!initialized || global_disable))
1517                 return;
1518
1519         ref.type         = dma_debug_single;
1520         ref.dev          = dev;
1521         ref.dev_addr     = dma_handle;
1522         ref.size         = size;
1523         ref.direction    = direction;
1524         ref.sg_call_ents = 0;
1525
1526         check_sync(dev, &ref, false);
1527 }
1528 EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1529
1530 void debug_dma_sync_single_range_for_cpu(struct device *dev,
1531                                          dma_addr_t dma_handle,
1532                                          unsigned long offset, size_t size,
1533                                          int direction)
1534 {
1535         struct dma_debug_entry ref;
1536
1537         if (unlikely(!initialized || global_disable))
1538                 return;
1539
1540         ref.type         = dma_debug_single;
1541         ref.dev          = dev;
1542         ref.dev_addr     = dma_handle;
1543         ref.size         = offset + size;
1544         ref.direction    = direction;
1545         ref.sg_call_ents = 0;
1546
1547         check_sync(dev, &ref, true);
1548 }
1549 EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1550
1551 void debug_dma_sync_single_range_for_device(struct device *dev,
1552                                             dma_addr_t dma_handle,
1553                                             unsigned long offset,
1554                                             size_t size, int direction)
1555 {
1556         struct dma_debug_entry ref;
1557
1558         if (unlikely(!initialized || global_disable))
1559                 return;
1560
1561         ref.type         = dma_debug_single;
1562         ref.dev          = dev;
1563         ref.dev_addr     = dma_handle;
1564         ref.size         = offset + size;
1565         ref.direction    = direction;
1566         ref.sg_call_ents = 0;
1567
1568         check_sync(dev, &ref, false);
1569 }
1570 EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
1571
1572 void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1573                                int nelems, int direction)
1574 {
1575         struct scatterlist *s;
1576         int mapped_ents = 0, i;
1577
1578         if (unlikely(!initialized || global_disable))
1579                 return;
1580
1581         for_each_sg(sg, s, nelems, i) {
1582
1583                 struct dma_debug_entry ref = {
1584                         .type           = dma_debug_sg,
1585                         .dev            = dev,
1586                         .paddr          = sg_phys(s),
1587                         .dev_addr       = sg_dma_address(s),
1588                         .size           = sg_dma_len(s),
1589                         .direction      = direction,
1590                         .sg_call_ents   = nelems,
1591                 };
1592
1593                 if (!i)
1594                         mapped_ents = get_nr_mapped_entries(dev, &ref);
1595
1596                 if (i >= mapped_ents)
1597                         break;
1598
1599                 check_sync(dev, &ref, true);
1600         }
1601 }
1602 EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
1603
1604 void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1605                                   int nelems, int direction)
1606 {
1607         struct scatterlist *s;
1608         int mapped_ents = 0, i;
1609
1610         if (unlikely(!initialized || global_disable))
1611                 return;
1612
1613         for_each_sg(sg, s, nelems, i) {
1614
1615                 struct dma_debug_entry ref = {
1616                         .type           = dma_debug_sg,
1617                         .dev            = dev,
1618                         .paddr          = sg_phys(s),
1619                         .dev_addr       = sg_dma_address(s),
1620                         .size           = sg_dma_len(s),
1621                         .direction      = direction,
1622                         .sg_call_ents   = nelems,
1623                 };
1624                 if (!i)
1625                         mapped_ents = get_nr_mapped_entries(dev, &ref);
1626
1627                 if (i >= mapped_ents)
1628                         break;
1629
1630                 check_sync(dev, &ref, false);
1631         }
1632 }
1633 EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
1634
1635 static int __init dma_debug_driver_setup(char *str)
1636 {
1637         int i;
1638
1639         for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1640                 current_driver_name[i] = *str;
1641                 if (*str == 0)
1642                         break;
1643         }
1644
1645         if (current_driver_name[0])
1646                 pr_info("DMA-API: enable driver filter for driver [%s]\n",
1647                         current_driver_name);
1648
1649
1650         return 1;
1651 }
1652 __setup("dma_debug_driver=", dma_debug_driver_setup);