misc: tegra-profiler: use power-of-2 sized buffer
[linux-3.10.git] / drivers / misc / tegra-profiler / comm.c
1 /*
2  * drivers/misc/tegra-profiler/comm.c
3  *
4  * Copyright (c) 2013-2017, NVIDIA CORPORATION.  All rights reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms and conditions of the GNU General Public License,
8  * version 2, as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope it will be useful, but WITHOUT
11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13  * more details.
14  *
15  */
16
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18
19 #include <linux/fs.h>
20 #include <linux/slab.h>
21 #include <linux/vmalloc.h>
22 #include <linux/miscdevice.h>
23 #include <linux/sched.h>
24 #include <linux/bitops.h>
25 #include <linux/err.h>
26 #include <linux/mm.h>
27 #include <linux/circ_buf.h>
28 #include <linux/uaccess.h>
29
30 #include <linux/tegra_profiler.h>
31
32 #include "comm.h"
33 #include "version.h"
34
35 struct quadd_ring_buffer {
36         struct quadd_ring_buffer_hdr *rb_hdr;
37         char *buf;
38
39         size_t max_fill_count;
40         size_t nr_skipped_samples;
41
42         struct quadd_mmap_area *mmap;
43
44         raw_spinlock_t lock;
45 };
46
47 struct quadd_comm_ctx {
48         struct quadd_comm_control_interface *control;
49
50         atomic_t active;
51
52         struct mutex io_mutex;
53         int nr_users;
54
55         int params_ok;
56
57         struct miscdevice *misc_dev;
58
59         struct list_head mmap_areas;
60         spinlock_t mmaps_lock;
61 };
62
63 struct comm_cpu_context {
64         struct quadd_ring_buffer rb;
65         int params_ok;
66 };
67
68 static struct quadd_comm_ctx comm_ctx;
69 static DEFINE_PER_CPU(struct comm_cpu_context, cpu_ctx);
70
71 static void
72 rb_write(struct quadd_ring_buffer_hdr *rb_hdr,
73          char *buf, const void *data, size_t length)
74 {
75         size_t len, head = rb_hdr->pos_write;
76         const char *s = data;
77
78         len = min_t(size_t, rb_hdr->size - head, length);
79
80         memcpy(buf + head, s, len);
81         head = (head + len) & (rb_hdr->size - 1);
82
83         if (length > len) {
84                 s += len;
85                 len = length - len;
86
87                 memcpy(buf + head, s, len);
88                 head += len;
89         }
90
91         rb_hdr->pos_write = head;
92 }
93
94 static ssize_t
95 write_sample(struct quadd_ring_buffer *rb,
96              struct quadd_record_data *sample,
97              const struct quadd_iovec *vec, int vec_count)
98 {
99         int i;
100         size_t len = 0, c;
101         struct quadd_ring_buffer_hdr hdr, *rb_hdr = rb->rb_hdr;
102
103         if (!rb_hdr)
104                 return -EIO;
105
106         if (vec) {
107                 for (i = 0; i < vec_count; i++)
108                         len += vec[i].len;
109         }
110
111         sample->extra_size = len;
112         len += sizeof(*sample);
113
114         hdr.size = rb_hdr->size;
115         hdr.pos_write = rb_hdr->pos_write;
116         hdr.pos_read = ACCESS_ONCE(rb_hdr->pos_read);
117
118         c = CIRC_SPACE(hdr.pos_write, hdr.pos_read, hdr.size);
119         if (len > c) {
120                 pr_err_once("[cpu: %d] warning: buffer has been overflowed\n",
121                             smp_processor_id());
122                 return -ENOSPC;
123         }
124
125         rb_write(&hdr, rb->buf, sample, sizeof(*sample));
126
127         if (vec) {
128                 for (i = 0; i < vec_count; i++)
129                         rb_write(&hdr, rb->buf, vec[i].base, vec[i].len);
130         }
131
132         c = CIRC_CNT(hdr.pos_write, hdr.pos_read, hdr.size);
133         if (c > rb->max_fill_count) {
134                 rb->max_fill_count = c;
135                 rb_hdr->max_fill_count = c;
136         }
137
138         smp_store_release(&rb_hdr->pos_write, hdr.pos_write);
139
140         return len;
141 }
142
143 static size_t get_data_size(void)
144 {
145         int cpu_id;
146         unsigned long flags;
147         size_t size = 0, tail;
148         struct comm_cpu_context *cc;
149         struct quadd_ring_buffer *rb;
150         struct quadd_ring_buffer_hdr *rb_hdr;
151
152         for_each_possible_cpu(cpu_id) {
153                 cc = &per_cpu(cpu_ctx, cpu_id);
154
155                 rb = &cc->rb;
156
157                 rb_hdr = rb->rb_hdr;
158                 if (!rb_hdr)
159                         continue;
160
161                 raw_spin_lock_irqsave(&rb->lock, flags);
162
163                 tail = ACCESS_ONCE(rb_hdr->pos_read);
164                 size += CIRC_CNT(rb_hdr->pos_write, tail, rb_hdr->size);
165
166                 raw_spin_unlock_irqrestore(&rb->lock, flags);
167         }
168
169         return size;
170 }
171
172 static ssize_t
173 put_sample(struct quadd_record_data *data,
174            struct quadd_iovec *vec,
175            int vec_count, int cpu_id)
176 {
177         ssize_t err = 0;
178         unsigned long flags;
179         struct comm_cpu_context *cc;
180         struct quadd_ring_buffer *rb;
181         struct quadd_ring_buffer_hdr *rb_hdr;
182
183         if (!atomic_read(&comm_ctx.active))
184                 return -EIO;
185
186         cc = cpu_id < 0 ? &__get_cpu_var(cpu_ctx) :
187                 &per_cpu(cpu_ctx, cpu_id);
188
189         rb = &cc->rb;
190
191         raw_spin_lock_irqsave(&rb->lock, flags);
192
193         err = write_sample(rb, data, vec, vec_count);
194         if (err < 0) {
195                 pr_err_once("%s: error: write sample\n", __func__);
196                 rb->nr_skipped_samples++;
197
198                 rb_hdr = rb->rb_hdr;
199                 if (rb_hdr)
200                         rb_hdr->skipped_samples++;
201         }
202
203         raw_spin_unlock_irqrestore(&rb->lock, flags);
204
205         return err;
206 }
207
208 static void comm_reset(void)
209 {
210         pr_debug("Comm reset\n");
211 }
212
213 static int is_active(void)
214 {
215         return atomic_read(&comm_ctx.active) != 0;
216 }
217
218 static struct quadd_comm_data_interface comm_data = {
219         .put_sample = put_sample,
220         .reset = comm_reset,
221         .is_active = is_active,
222 };
223
224 static struct quadd_mmap_area *
225 find_mmap(unsigned long vm_start)
226 {
227         struct quadd_mmap_area *entry;
228
229         list_for_each_entry(entry, &comm_ctx.mmap_areas, list) {
230                 struct vm_area_struct *mmap_vma = entry->mmap_vma;
231
232                 if (vm_start == mmap_vma->vm_start)
233                         return entry;
234         }
235
236         return NULL;
237 }
238
239 static int device_open(struct inode *inode, struct file *file)
240 {
241         mutex_lock(&comm_ctx.io_mutex);
242         comm_ctx.nr_users++;
243         mutex_unlock(&comm_ctx.io_mutex);
244         return 0;
245 }
246
247 static int device_release(struct inode *inode, struct file *file)
248 {
249         mutex_lock(&comm_ctx.io_mutex);
250         comm_ctx.nr_users--;
251
252         if (comm_ctx.nr_users == 0) {
253                 if (atomic_cmpxchg(&comm_ctx.active, 1, 0)) {
254                         comm_ctx.control->stop();
255                         pr_info("Stop profiling: daemon is closed\n");
256                 }
257         }
258         mutex_unlock(&comm_ctx.io_mutex);
259
260         return 0;
261 }
262
263 static int
264 init_mmap_hdr(struct quadd_mmap_rb_info *mmap_rb,
265               struct quadd_mmap_area *mmap)
266 {
267         unsigned int cpu_id;
268         size_t size;
269         unsigned long flags;
270         struct vm_area_struct *vma;
271         struct quadd_ring_buffer *rb;
272         struct quadd_ring_buffer_hdr *rb_hdr;
273         struct quadd_mmap_header *mmap_hdr;
274         struct comm_cpu_context *cc;
275
276         if (mmap->type != QUADD_MMAP_TYPE_RB)
277                 return -EIO;
278
279         cpu_id = mmap_rb->cpu_id;
280
281         if (cpu_id >= nr_cpu_ids)
282                 return -EINVAL;
283
284         cc = &per_cpu(cpu_ctx, cpu_id);
285
286         rb = &cc->rb;
287
288         vma = mmap->mmap_vma;
289         size = vma->vm_end - vma->vm_start;
290
291         if (size <= PAGE_SIZE || !is_power_of_2(size - PAGE_SIZE))
292                 return -EINVAL;
293
294         size -= PAGE_SIZE;
295
296         raw_spin_lock_irqsave(&rb->lock, flags);
297
298         mmap->rb = rb;
299
300         rb->mmap = mmap;
301         rb->buf = (char *)mmap->data + PAGE_SIZE;
302
303         rb->max_fill_count = 0;
304         rb->nr_skipped_samples = 0;
305
306         mmap_hdr = mmap->data;
307
308         mmap_hdr->magic = QUADD_MMAP_HEADER_MAGIC;
309         mmap_hdr->version = QUADD_MMAP_HEADER_VERSION;
310         mmap_hdr->cpu_id = cpu_id;
311         mmap_hdr->samples_version = QUADD_SAMPLES_VERSION;
312
313         rb_hdr = (struct quadd_ring_buffer_hdr *)(mmap_hdr + 1);
314         rb->rb_hdr = rb_hdr;
315
316         rb_hdr->size = size;
317         rb_hdr->pos_read = 0;
318         rb_hdr->pos_write = 0;
319
320         rb_hdr->max_fill_count = 0;
321         rb_hdr->skipped_samples = 0;
322
323         rb_hdr->state = QUADD_RB_STATE_ACTIVE;
324
325         raw_spin_unlock_irqrestore(&rb->lock, flags);
326
327         pr_info("[cpu: %d] init_mmap_hdr: vma: %#lx - %#lx, data: %p - %p\n",
328                 cpu_id,
329                 vma->vm_start, vma->vm_end,
330                 mmap->data, mmap->data + vma->vm_end - vma->vm_start);
331
332         return 0;
333 }
334
335 static void rb_stop(void)
336 {
337         int cpu_id;
338         struct quadd_ring_buffer *rb;
339         struct quadd_ring_buffer_hdr *rb_hdr;
340         struct comm_cpu_context *cc;
341
342         for_each_possible_cpu(cpu_id) {
343                 cc = &per_cpu(cpu_ctx, cpu_id);
344
345                 rb = &cc->rb;
346                 rb_hdr = rb->rb_hdr;
347
348                 if (!rb_hdr)
349                         continue;
350
351                 pr_info("[%d] skipped samples/max filling: %zu/%zu\n",
352                         cpu_id, rb->nr_skipped_samples, rb->max_fill_count);
353
354                 rb_hdr->state = QUADD_RB_STATE_STOPPED;
355         }
356 }
357
358 static void rb_reset(struct quadd_ring_buffer *rb)
359 {
360         unsigned long flags;
361
362         if (!rb)
363                 return;
364
365         raw_spin_lock_irqsave(&rb->lock, flags);
366
367         rb->mmap = NULL;
368         rb->buf = NULL;
369         rb->rb_hdr = NULL;
370
371         raw_spin_unlock_irqrestore(&rb->lock, flags);
372 }
373
374 static int
375 ready_to_profile(void)
376 {
377         int cpuid, is_cpu_present;
378         struct comm_cpu_context *cc;
379
380         if (!comm_ctx.params_ok)
381                 return 0;
382
383         for_each_possible_cpu(cpuid) {
384                 is_cpu_present = comm_ctx.control->is_cpu_present(cpuid);
385
386                 if (is_cpu_present) {
387                         cc = &per_cpu(cpu_ctx, cpuid);
388
389                         if (!cc->params_ok)
390                                 return 0;
391                 }
392         }
393
394         return 1;
395 }
396
397 static void
398 reset_params_ok_flag(void)
399 {
400         int cpu_id;
401
402         comm_ctx.params_ok = 0;
403         for_each_possible_cpu(cpu_id) {
404                 struct comm_cpu_context *cc = &per_cpu(cpu_ctx, cpu_id);
405
406                 cc->params_ok = 0;
407         }
408 }
409
410 static long
411 device_ioctl(struct file *file,
412              unsigned int ioctl_num,
413              unsigned long ioctl_param)
414 {
415         int err = 0;
416         unsigned int cpuid;
417         struct quadd_mmap_area *mmap;
418         struct quadd_parameters *user_params;
419         struct quadd_pmu_setup_for_cpu *cpu_pmu_params;
420         struct quadd_comm_cap_for_cpu *per_cpu_cap;
421         struct quadd_comm_cap cap;
422         struct quadd_module_state state;
423         struct quadd_module_version versions;
424         struct quadd_sections extabs;
425         struct quadd_mmap_rb_info mmap_rb;
426
427         mutex_lock(&comm_ctx.io_mutex);
428
429         if (ioctl_num != IOCTL_SETUP &&
430             ioctl_num != IOCTL_GET_CAP &&
431             ioctl_num != IOCTL_GET_STATE &&
432             ioctl_num != IOCTL_SETUP_PMU_FOR_CPU &&
433             ioctl_num != IOCTL_GET_CAP_FOR_CPU &&
434             ioctl_num != IOCTL_GET_VERSION) {
435                 if (!ready_to_profile()) {
436                         err = -EACCES;
437                         goto error_out;
438                 }
439         }
440
441         switch (ioctl_num) {
442         case IOCTL_SETUP_PMU_FOR_CPU:
443                 if (atomic_read(&comm_ctx.active)) {
444                         pr_err("error: tegra profiler is active\n");
445                         err = -EBUSY;
446                         goto error_out;
447                 }
448
449                 cpu_pmu_params = vmalloc(sizeof(*cpu_pmu_params));
450                 if (!cpu_pmu_params) {
451                         err = -ENOMEM;
452                         goto error_out;
453                 }
454
455                 if (copy_from_user(cpu_pmu_params,
456                                    (void __user *)ioctl_param,
457                                    sizeof(*cpu_pmu_params))) {
458                         pr_err("setup failed\n");
459                         vfree(cpu_pmu_params);
460                         err = -EFAULT;
461                         goto error_out;
462                 }
463
464                 cpuid = cpu_pmu_params->cpuid;
465
466                 if (cpuid >= nr_cpu_ids) {
467                         vfree(cpu_pmu_params);
468                         err = -EINVAL;
469                         goto error_out;
470                 }
471
472                 per_cpu(cpu_ctx, cpuid).params_ok = 0;
473
474                 err = comm_ctx.control->set_parameters_for_cpu(cpu_pmu_params);
475                 if (err) {
476                         pr_err("error: setup failed\n");
477                         vfree(cpu_pmu_params);
478                         goto error_out;
479                 }
480
481                 per_cpu(cpu_ctx, cpuid).params_ok = 1;
482
483                 pr_info("setup PMU success for cpu: %d\n", cpuid);
484
485                 vfree(cpu_pmu_params);
486                 break;
487
488         case IOCTL_SETUP:
489                 if (atomic_read(&comm_ctx.active)) {
490                         pr_err("error: tegra profiler is active\n");
491                         err = -EBUSY;
492                         goto error_out;
493                 }
494                 comm_ctx.params_ok = 0;
495
496                 user_params = vmalloc(sizeof(*user_params));
497                 if (!user_params) {
498                         err = -ENOMEM;
499                         goto error_out;
500                 }
501
502                 if (copy_from_user(user_params, (void __user *)ioctl_param,
503                                    sizeof(struct quadd_parameters))) {
504                         pr_err("setup failed\n");
505                         vfree(user_params);
506                         err = -EFAULT;
507                         goto error_out;
508                 }
509
510                 err = comm_ctx.control->set_parameters(user_params);
511                 if (err) {
512                         pr_err("error: setup failed\n");
513                         vfree(user_params);
514                         goto error_out;
515                 }
516
517                 if (user_params->reserved[QUADD_PARAM_IDX_SIZE_OF_RB] == 0) {
518                         pr_err("error: too old version of daemon\n");
519                         vfree(user_params);
520                         err = -EINVAL;
521                         goto error_out;
522                 }
523
524                 comm_ctx.params_ok = 1;
525
526                 pr_info("setup success: freq/mafreq: %u/%u, backtrace: %d, pid: %d\n",
527                         user_params->freq,
528                         user_params->ma_freq,
529                         user_params->backtrace,
530                         user_params->pids[0]);
531
532                 vfree(user_params);
533                 break;
534
535         case IOCTL_GET_CAP:
536                 memset(&cap, 0, sizeof(cap));
537                 comm_ctx.control->get_capabilities(&cap);
538                 if (copy_to_user((void __user *)ioctl_param, &cap,
539                                  sizeof(struct quadd_comm_cap))) {
540                         pr_err("error: get_capabilities failed\n");
541                         err = -EFAULT;
542                         goto error_out;
543                 }
544                 break;
545
546         case IOCTL_GET_CAP_FOR_CPU:
547                 per_cpu_cap = vmalloc(sizeof(*per_cpu_cap));
548                 if (!per_cpu_cap) {
549                         err = -ENOMEM;
550                         goto error_out;
551                 }
552
553                 if (copy_from_user(per_cpu_cap, (void __user *)ioctl_param,
554                                    sizeof(*per_cpu_cap))) {
555                         pr_err("setup failed\n");
556                         vfree(per_cpu_cap);
557                         err = -EFAULT;
558                         goto error_out;
559                 }
560
561                 cpuid = per_cpu_cap->cpuid;
562
563                 if (cpuid >= nr_cpu_ids) {
564                         vfree(per_cpu_cap);
565                         err = -EINVAL;
566                         goto error_out;
567                 }
568
569                 comm_ctx.control->get_capabilities_for_cpu(cpuid, per_cpu_cap);
570
571                 if (copy_to_user((void __user *)ioctl_param, per_cpu_cap,
572                                  sizeof(*per_cpu_cap))) {
573                         pr_err("error: get_capabilities failed\n");
574                         vfree(per_cpu_cap);
575                         err = -EFAULT;
576                         goto error_out;
577                 }
578
579                 vfree(per_cpu_cap);
580                 break;
581
582         case IOCTL_GET_VERSION:
583                 memset(&versions, 0, sizeof(versions));
584
585                 strlcpy((char *)versions.branch, QUADD_MODULE_BRANCH,
586                         sizeof(versions.branch));
587                 strlcpy((char *)versions.version, QUADD_MODULE_VERSION,
588                         sizeof(versions.version));
589
590                 versions.samples_version = QUADD_SAMPLES_VERSION;
591                 versions.io_version = QUADD_IO_VERSION;
592
593                 if (copy_to_user((void __user *)ioctl_param, &versions,
594                                  sizeof(struct quadd_module_version))) {
595                         pr_err("error: get version failed\n");
596                         err = -EFAULT;
597                         goto error_out;
598                 }
599                 break;
600
601         case IOCTL_GET_STATE:
602                 comm_ctx.control->get_state(&state);
603
604                 state.buffer_size = 0;
605                 state.buffer_fill_size = get_data_size();
606                 state.reserved[QUADD_MOD_STATE_IDX_RB_MAX_FILL_COUNT] = 0;
607
608                 if (copy_to_user((void __user *)ioctl_param, &state,
609                                  sizeof(struct quadd_module_state))) {
610                         pr_err("error: get_state failed\n");
611                         err = -EFAULT;
612                         goto error_out;
613                 }
614                 break;
615
616         case IOCTL_START:
617                 if (!atomic_cmpxchg(&comm_ctx.active, 0, 1)) {
618                         err = comm_ctx.control->start();
619                         if (err) {
620                                 pr_err("error: start failed\n");
621                                 atomic_set(&comm_ctx.active, 0);
622                                 goto error_out;
623                         }
624                         pr_info("Start profiling success\n");
625                 }
626                 break;
627
628         case IOCTL_STOP:
629                 if (atomic_cmpxchg(&comm_ctx.active, 1, 0)) {
630                         reset_params_ok_flag();
631                         comm_ctx.control->stop();
632                         rb_stop();
633                         pr_info("Stop profiling success\n");
634                 }
635                 break;
636
637         case IOCTL_SET_SECTIONS_INFO:
638                 if (copy_from_user(&extabs, (void __user *)ioctl_param,
639                                    sizeof(extabs))) {
640                         pr_err("error: set_sections_info failed\n");
641                         err = -EFAULT;
642                         goto error_out;
643                 }
644
645                 pr_debug("%s: user_mmap_start: %#llx, sections vma: %#llx - %#llx\n",
646                          __func__,
647                          (unsigned long long)extabs.user_mmap_start,
648                          (unsigned long long)extabs.vm_start,
649                          (unsigned long long)extabs.vm_end);
650
651                 spin_lock(&comm_ctx.mmaps_lock);
652                 mmap = find_mmap(extabs.user_mmap_start);
653                 if (!mmap) {
654                         pr_err("%s: error: mmap is not found\n", __func__);
655                         err = -ENXIO;
656                         spin_unlock(&comm_ctx.mmaps_lock);
657                         goto error_out;
658                 }
659
660                 mmap->type = QUADD_MMAP_TYPE_EXTABS;
661                 mmap->rb = NULL;
662
663                 err = comm_ctx.control->set_extab(&extabs, mmap);
664                 spin_unlock(&comm_ctx.mmaps_lock);
665                 if (err) {
666                         pr_err("error: set_sections_info\n");
667                         goto error_out;
668                 }
669                 break;
670
671         case IOCTL_SET_MMAP_RB:
672                 if (copy_from_user(&mmap_rb, (void __user *)ioctl_param,
673                                    sizeof(mmap_rb))) {
674                         pr_err("%s: error: mmap_rb failed\n", __func__);
675                         err = -EFAULT;
676                         goto error_out;
677                 }
678
679                 spin_lock(&comm_ctx.mmaps_lock);
680                 mmap = find_mmap((unsigned long)mmap_rb.vm_start);
681                 spin_unlock(&comm_ctx.mmaps_lock);
682                 if (!mmap) {
683                         pr_err("%s: error: mmap is not found\n", __func__);
684                         err = -ENXIO;
685                         goto error_out;
686                 }
687                 mmap->type = QUADD_MMAP_TYPE_RB;
688
689                 err = init_mmap_hdr(&mmap_rb, mmap);
690                 if (err) {
691                         pr_err("%s: error: init_mmap_hdr\n", __func__);
692                         goto error_out;
693                 }
694
695                 break;
696
697         default:
698                 pr_err("error: ioctl %u is unsupported in this version of module\n",
699                        ioctl_num);
700                 err = -EFAULT;
701                 goto error_out;
702         }
703
704 error_out:
705         mutex_unlock(&comm_ctx.io_mutex);
706         return err;
707 }
708
709 static void
710 delete_mmap(struct quadd_mmap_area *mmap)
711 {
712         struct quadd_mmap_area *entry, *next;
713
714         list_for_each_entry_safe(entry, next, &comm_ctx.mmap_areas, list) {
715                 if (entry == mmap) {
716                         list_del(&entry->list);
717                         vfree(entry->data);
718                         kfree(entry);
719                         break;
720                 }
721         }
722 }
723
724 static void mmap_open(struct vm_area_struct *vma)
725 {
726         pr_debug("%s: mmap_open: vma: %#lx - %#lx\n",
727                 __func__, vma->vm_start, vma->vm_end);
728 }
729
730 static void mmap_close(struct vm_area_struct *vma)
731 {
732         struct quadd_mmap_area *mmap;
733
734         pr_debug("%s: mmap_close: vma: %#lx - %#lx\n",
735                  __func__, vma->vm_start, vma->vm_end);
736
737         spin_lock(&comm_ctx.mmaps_lock);
738
739         mmap = find_mmap(vma->vm_start);
740         if (!mmap) {
741                 pr_err("%s: error: mmap is not found\n", __func__);
742                 goto out;
743         }
744
745         pr_debug("mmap_close: type: %d\n", mmap->type);
746
747         if (mmap->type == QUADD_MMAP_TYPE_EXTABS)
748                 comm_ctx.control->delete_mmap(mmap);
749         else if (mmap->type == QUADD_MMAP_TYPE_RB)
750                 rb_reset(mmap->rb);
751         else
752                 pr_err("error: mmap area is uninitialized\n");
753
754         delete_mmap(mmap);
755
756 out:
757         spin_unlock(&comm_ctx.mmaps_lock);
758 }
759
760 static int mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
761 {
762         void *data;
763         struct quadd_mmap_area *mmap;
764         unsigned long offset = vmf->pgoff << PAGE_SHIFT;
765
766         pr_debug("mmap_fault: vma: %#lx - %#lx, pgoff: %#lx, vaddr: %p\n",
767                  vma->vm_start, vma->vm_end, vmf->pgoff, vmf->virtual_address);
768
769         spin_lock(&comm_ctx.mmaps_lock);
770
771         mmap = find_mmap(vma->vm_start);
772         if (!mmap) {
773                 spin_unlock(&comm_ctx.mmaps_lock);
774                 return VM_FAULT_SIGBUS;
775         }
776
777         data = mmap->data;
778
779         vmf->page = vmalloc_to_page(data + offset);
780         get_page(vmf->page);
781
782         spin_unlock(&comm_ctx.mmaps_lock);
783         return 0;
784 }
785
786 static struct vm_operations_struct mmap_vm_ops = {
787         .open   = mmap_open,
788         .close  = mmap_close,
789         .fault  = mmap_fault,
790 };
791
792 static int
793 device_mmap(struct file *filp, struct vm_area_struct *vma)
794 {
795         unsigned long vma_size, nr_pages;
796         struct quadd_mmap_area *entry;
797
798         pr_debug("mmap: vma: %#lx - %#lx, pgoff: %#lx\n",
799                  vma->vm_start, vma->vm_end, vma->vm_pgoff);
800
801         if (vma->vm_pgoff != 0)
802                 return -EINVAL;
803
804         vma->vm_private_data = filp->private_data;
805
806         vma_size = vma->vm_end - vma->vm_start;
807         nr_pages = vma_size / PAGE_SIZE;
808
809         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
810         if (!entry)
811                 return -ENOMEM;
812
813         entry->mmap_vma = vma;
814
815         INIT_LIST_HEAD(&entry->list);
816         INIT_LIST_HEAD(&entry->ex_entries);
817
818         entry->data = vmalloc_user(nr_pages * PAGE_SIZE);
819         if (!entry->data) {
820                 pr_err("%s: error: vmalloc_user", __func__);
821                 kfree(entry);
822                 return -ENOMEM;
823         }
824
825         entry->type = QUADD_MMAP_TYPE_NONE;
826
827         pr_debug("%s: data: %p - %p (%#lx)\n",
828                  __func__, entry->data, entry->data + nr_pages * PAGE_SIZE,
829                  nr_pages * PAGE_SIZE);
830
831         spin_lock(&comm_ctx.mmaps_lock);
832         list_add_tail(&entry->list, &comm_ctx.mmap_areas);
833         spin_unlock(&comm_ctx.mmaps_lock);
834
835         vma->vm_ops = &mmap_vm_ops;
836         vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
837
838         vma->vm_ops->open(vma);
839
840         return 0;
841 }
842
843 static void unregister(void)
844 {
845         misc_deregister(comm_ctx.misc_dev);
846         kfree(comm_ctx.misc_dev);
847 }
848
849 static const struct file_operations qm_fops = {
850         .open           = device_open,
851         .release        = device_release,
852         .unlocked_ioctl = device_ioctl,
853         .compat_ioctl   = device_ioctl,
854         .mmap           = device_mmap,
855 };
856
857 static int comm_init(void)
858 {
859         int res, cpu_id;
860         struct miscdevice *misc_dev;
861
862         misc_dev = kzalloc(sizeof(*misc_dev), GFP_KERNEL);
863         if (!misc_dev)
864                 return -ENOMEM;
865
866         misc_dev->minor = MISC_DYNAMIC_MINOR;
867         misc_dev->name = QUADD_DEVICE_NAME;
868         misc_dev->fops = &qm_fops;
869
870         res = misc_register(misc_dev);
871         if (res < 0) {
872                 pr_err("Error: misc_register: %d\n", res);
873                 kfree(misc_dev);
874                 return res;
875         }
876         comm_ctx.misc_dev = misc_dev;
877
878         mutex_init(&comm_ctx.io_mutex);
879         atomic_set(&comm_ctx.active, 0);
880
881         comm_ctx.nr_users = 0;
882
883         INIT_LIST_HEAD(&comm_ctx.mmap_areas);
884         spin_lock_init(&comm_ctx.mmaps_lock);
885
886         for_each_possible_cpu(cpu_id) {
887                 struct comm_cpu_context *cc = &per_cpu(cpu_ctx, cpu_id);
888                 struct quadd_ring_buffer *rb = &cc->rb;
889
890                 rb->mmap = NULL;
891                 rb->buf = NULL;
892                 rb->rb_hdr = NULL;
893
894                 rb->max_fill_count = 0;
895                 rb->nr_skipped_samples = 0;
896
897                 raw_spin_lock_init(&rb->lock);
898         }
899
900         reset_params_ok_flag();
901
902         return 0;
903 }
904
905 struct quadd_comm_data_interface *
906 quadd_comm_events_init(struct quadd_comm_control_interface *control)
907 {
908         int err;
909
910         err = comm_init();
911         if (err < 0)
912                 return ERR_PTR(err);
913
914         comm_ctx.control = control;
915         return &comm_data;
916 }
917
918 void quadd_comm_events_exit(void)
919 {
920         mutex_lock(&comm_ctx.io_mutex);
921         unregister();
922         mutex_unlock(&comm_ctx.io_mutex);
923 }