Merge branch 'core-iommu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-2.6.git] / arch / s390 / oprofile / hwsampler.c
1 /**
2  * arch/s390/oprofile/hwsampler.c
3  *
4  * Copyright IBM Corp. 2010
5  * Author: Heinz Graalfs <graalfs@de.ibm.com>
6  */
7
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/smp.h>
11 #include <linux/errno.h>
12 #include <linux/workqueue.h>
13 #include <linux/interrupt.h>
14 #include <linux/notifier.h>
15 #include <linux/cpu.h>
16 #include <linux/semaphore.h>
17 #include <linux/oom.h>
18 #include <linux/oprofile.h>
19
20 #include <asm/lowcore.h>
21 #include <asm/s390_ext.h>
22
23 #include "hwsampler.h"
24
25 #define MAX_NUM_SDB 511
26 #define MIN_NUM_SDB 1
27
28 #define ALERT_REQ_MASK   0x4000000000000000ul
29 #define BUFFER_FULL_MASK 0x8000000000000000ul
30
31 #define EI_IEA      (1 << 31)   /* invalid entry address              */
32 #define EI_ISE      (1 << 30)   /* incorrect SDBT entry               */
33 #define EI_PRA      (1 << 29)   /* program request alert              */
34 #define EI_SACA     (1 << 23)   /* sampler authorization change alert */
35 #define EI_LSDA     (1 << 22)   /* loss of sample data alert          */
36
37 DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
38
39 struct hws_execute_parms {
40         void *buffer;
41         signed int rc;
42 };
43
44 DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
45 EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
46
47 static DEFINE_MUTEX(hws_sem);
48 static DEFINE_MUTEX(hws_sem_oom);
49
50 static unsigned char hws_flush_all;
51 static unsigned int hws_oom;
52 static struct workqueue_struct *hws_wq;
53
54 static unsigned int hws_state;
55 enum {
56         HWS_INIT = 1,
57         HWS_DEALLOCATED,
58         HWS_STOPPED,
59         HWS_STARTED,
60         HWS_STOPPING };
61
62 /* set to 1 if called by kernel during memory allocation */
63 static unsigned char oom_killer_was_active;
64 /* size of SDBT and SDB as of allocate API */
65 static unsigned long num_sdbt = 100;
66 static unsigned long num_sdb = 511;
67 /* sampling interval (machine cycles) */
68 static unsigned long interval;
69
70 static unsigned long min_sampler_rate;
71 static unsigned long max_sampler_rate;
72
73 static int ssctl(void *buffer)
74 {
75         int cc;
76
77         /* set in order to detect a program check */
78         cc = 1;
79
80         asm volatile(
81                 "0: .insn s,0xB2870000,0(%1)\n"
82                 "1: ipm %0\n"
83                 "   srl %0,28\n"
84                 "2:\n"
85                 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
86                 : "+d" (cc), "+a" (buffer)
87                 : "m" (*((struct hws_ssctl_request_block *)buffer))
88                 : "cc", "memory");
89
90         return cc ? -EINVAL : 0 ;
91 }
92
93 static int qsi(void *buffer)
94 {
95         int cc;
96         cc = 1;
97
98         asm volatile(
99                 "0: .insn s,0xB2860000,0(%1)\n"
100                 "1: lhi %0,0\n"
101                 "2:\n"
102                 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
103                 : "=d" (cc), "+a" (buffer)
104                 : "m" (*((struct hws_qsi_info_block *)buffer))
105                 : "cc", "memory");
106
107         return cc ? -EINVAL : 0;
108 }
109
110 static void execute_qsi(void *parms)
111 {
112         struct hws_execute_parms *ep = parms;
113
114         ep->rc = qsi(ep->buffer);
115 }
116
117 static void execute_ssctl(void *parms)
118 {
119         struct hws_execute_parms *ep = parms;
120
121         ep->rc = ssctl(ep->buffer);
122 }
123
124 static int smp_ctl_ssctl_stop(int cpu)
125 {
126         int rc;
127         struct hws_execute_parms ep;
128         struct hws_cpu_buffer *cb;
129
130         cb = &per_cpu(sampler_cpu_buffer, cpu);
131
132         cb->ssctl.es = 0;
133         cb->ssctl.cs = 0;
134
135         ep.buffer = &cb->ssctl;
136         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
137         rc = ep.rc;
138         if (rc) {
139                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
140                 dump_stack();
141         }
142
143         ep.buffer = &cb->qsi;
144         smp_call_function_single(cpu, execute_qsi, &ep, 1);
145
146         if (cb->qsi.es || cb->qsi.cs) {
147                 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
148                 dump_stack();
149         }
150
151         return rc;
152 }
153
154 static int smp_ctl_ssctl_deactivate(int cpu)
155 {
156         int rc;
157         struct hws_execute_parms ep;
158         struct hws_cpu_buffer *cb;
159
160         cb = &per_cpu(sampler_cpu_buffer, cpu);
161
162         cb->ssctl.es = 1;
163         cb->ssctl.cs = 0;
164
165         ep.buffer = &cb->ssctl;
166         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
167         rc = ep.rc;
168         if (rc)
169                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
170
171         ep.buffer = &cb->qsi;
172         smp_call_function_single(cpu, execute_qsi, &ep, 1);
173
174         if (cb->qsi.cs)
175                 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
176
177         return rc;
178 }
179
180 static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
181 {
182         int rc;
183         struct hws_execute_parms ep;
184         struct hws_cpu_buffer *cb;
185
186         cb = &per_cpu(sampler_cpu_buffer, cpu);
187
188         cb->ssctl.h = 1;
189         cb->ssctl.tear = cb->first_sdbt;
190         cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
191         cb->ssctl.interval = interval;
192         cb->ssctl.es = 1;
193         cb->ssctl.cs = 1;
194
195         ep.buffer = &cb->ssctl;
196         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
197         rc = ep.rc;
198         if (rc)
199                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
200
201         ep.buffer = &cb->qsi;
202         smp_call_function_single(cpu, execute_qsi, &ep, 1);
203         if (ep.rc)
204                 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
205
206         return rc;
207 }
208
209 static int smp_ctl_qsi(int cpu)
210 {
211         struct hws_execute_parms ep;
212         struct hws_cpu_buffer *cb;
213
214         cb = &per_cpu(sampler_cpu_buffer, cpu);
215
216         ep.buffer = &cb->qsi;
217         smp_call_function_single(cpu, execute_qsi, &ep, 1);
218
219         return ep.rc;
220 }
221
222 static inline unsigned long *trailer_entry_ptr(unsigned long v)
223 {
224         void *ret;
225
226         ret = (void *)v;
227         ret += PAGE_SIZE;
228         ret -= sizeof(struct hws_trailer_entry);
229
230         return (unsigned long *) ret;
231 }
232
233 /* prototypes for external interrupt handler and worker */
234 static void hws_ext_handler(unsigned int ext_int_code,
235                                 unsigned int param32, unsigned long param64);
236
237 static void worker(struct work_struct *work);
238
239 static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
240                                 unsigned long *dear);
241
242 static void init_all_cpu_buffers(void)
243 {
244         int cpu;
245         struct hws_cpu_buffer *cb;
246
247         for_each_online_cpu(cpu) {
248                 cb = &per_cpu(sampler_cpu_buffer, cpu);
249                 memset(cb, 0, sizeof(struct hws_cpu_buffer));
250         }
251 }
252
253 static int is_link_entry(unsigned long *s)
254 {
255         return *s & 0x1ul ? 1 : 0;
256 }
257
258 static unsigned long *get_next_sdbt(unsigned long *s)
259 {
260         return (unsigned long *) (*s & ~0x1ul);
261 }
262
263 static int prepare_cpu_buffers(void)
264 {
265         int cpu;
266         int rc;
267         struct hws_cpu_buffer *cb;
268
269         rc = 0;
270         for_each_online_cpu(cpu) {
271                 cb = &per_cpu(sampler_cpu_buffer, cpu);
272                 atomic_set(&cb->ext_params, 0);
273                 cb->worker_entry = 0;
274                 cb->sample_overflow = 0;
275                 cb->req_alert = 0;
276                 cb->incorrect_sdbt_entry = 0;
277                 cb->invalid_entry_address = 0;
278                 cb->loss_of_sample_data = 0;
279                 cb->sample_auth_change_alert = 0;
280                 cb->finish = 0;
281                 cb->oom = 0;
282                 cb->stop_mode = 0;
283         }
284
285         return rc;
286 }
287
288 /*
289  * allocate_sdbt() - allocate sampler memory
290  * @cpu: the cpu for which sampler memory is allocated
291  *
292  * A 4K page is allocated for each requested SDBT.
293  * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
294  * Set ALERT_REQ mask in each SDBs trailer.
295  * Returns zero if successful, <0 otherwise.
296  */
297 static int allocate_sdbt(int cpu)
298 {
299         int j, k, rc;
300         unsigned long *sdbt;
301         unsigned long  sdb;
302         unsigned long *tail;
303         unsigned long *trailer;
304         struct hws_cpu_buffer *cb;
305
306         cb = &per_cpu(sampler_cpu_buffer, cpu);
307
308         if (cb->first_sdbt)
309                 return -EINVAL;
310
311         sdbt = NULL;
312         tail = sdbt;
313
314         for (j = 0; j < num_sdbt; j++) {
315                 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
316
317                 mutex_lock(&hws_sem_oom);
318                 /* OOM killer might have been activated */
319                 barrier();
320                 if (oom_killer_was_active || !sdbt) {
321                         if (sdbt)
322                                 free_page((unsigned long)sdbt);
323
324                         goto allocate_sdbt_error;
325                 }
326                 if (cb->first_sdbt == 0)
327                         cb->first_sdbt = (unsigned long)sdbt;
328
329                 /* link current page to tail of chain */
330                 if (tail)
331                         *tail = (unsigned long)(void *)sdbt + 1;
332
333                 mutex_unlock(&hws_sem_oom);
334
335                 for (k = 0; k < num_sdb; k++) {
336                         /* get and set SDB page */
337                         sdb = get_zeroed_page(GFP_KERNEL);
338
339                         mutex_lock(&hws_sem_oom);
340                         /* OOM killer might have been activated */
341                         barrier();
342                         if (oom_killer_was_active || !sdb) {
343                                 if (sdb)
344                                         free_page(sdb);
345
346                                 goto allocate_sdbt_error;
347                         }
348                         *sdbt = sdb;
349                         trailer = trailer_entry_ptr(*sdbt);
350                         *trailer = ALERT_REQ_MASK;
351                         sdbt++;
352                         mutex_unlock(&hws_sem_oom);
353                 }
354                 tail = sdbt;
355         }
356         mutex_lock(&hws_sem_oom);
357         if (oom_killer_was_active)
358                 goto allocate_sdbt_error;
359
360         rc = 0;
361         if (tail)
362                 *tail = (unsigned long)
363                         ((void *)cb->first_sdbt) + 1;
364
365 allocate_sdbt_exit:
366         mutex_unlock(&hws_sem_oom);
367         return rc;
368
369 allocate_sdbt_error:
370         rc = -ENOMEM;
371         goto allocate_sdbt_exit;
372 }
373
374 /*
375  * deallocate_sdbt() - deallocate all sampler memory
376  *
377  * For each online CPU all SDBT trees are deallocated.
378  * Returns the number of freed pages.
379  */
380 static int deallocate_sdbt(void)
381 {
382         int cpu;
383         int counter;
384
385         counter = 0;
386
387         for_each_online_cpu(cpu) {
388                 unsigned long start;
389                 unsigned long sdbt;
390                 unsigned long *curr;
391                 struct hws_cpu_buffer *cb;
392
393                 cb = &per_cpu(sampler_cpu_buffer, cpu);
394
395                 if (!cb->first_sdbt)
396                         continue;
397
398                 sdbt = cb->first_sdbt;
399                 curr = (unsigned long *) sdbt;
400                 start = sdbt;
401
402                 /* we'll free the SDBT after all SDBs are processed... */
403                 while (1) {
404                         if (!*curr || !sdbt)
405                                 break;
406
407                         /* watch for link entry reset if found */
408                         if (is_link_entry(curr)) {
409                                 curr = get_next_sdbt(curr);
410                                 if (sdbt)
411                                         free_page(sdbt);
412
413                                 /* we are done if we reach the start */
414                                 if ((unsigned long) curr == start)
415                                         break;
416                                 else
417                                         sdbt = (unsigned long) curr;
418                         } else {
419                                 /* process SDB pointer */
420                                 if (*curr) {
421                                         free_page(*curr);
422                                         curr++;
423                                 }
424                         }
425                         counter++;
426                 }
427                 cb->first_sdbt = 0;
428         }
429         return counter;
430 }
431
432 static int start_sampling(int cpu)
433 {
434         int rc;
435         struct hws_cpu_buffer *cb;
436
437         cb = &per_cpu(sampler_cpu_buffer, cpu);
438         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
439         if (rc) {
440                 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
441                 goto start_exit;
442         }
443
444         rc = -EINVAL;
445         if (!cb->qsi.es) {
446                 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
447                 goto start_exit;
448         }
449
450         if (!cb->qsi.cs) {
451                 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
452                 goto start_exit;
453         }
454
455         printk(KERN_INFO
456                 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
457                 cpu, interval);
458
459         rc = 0;
460
461 start_exit:
462         return rc;
463 }
464
465 static int stop_sampling(int cpu)
466 {
467         unsigned long v;
468         int rc;
469         struct hws_cpu_buffer *cb;
470
471         rc = smp_ctl_qsi(cpu);
472         WARN_ON(rc);
473
474         cb = &per_cpu(sampler_cpu_buffer, cpu);
475         if (!rc && !cb->qsi.es)
476                 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
477
478         rc = smp_ctl_ssctl_stop(cpu);
479         if (rc) {
480                 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
481                                 cpu, rc);
482                 goto stop_exit;
483         }
484
485         printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
486
487 stop_exit:
488         v = cb->req_alert;
489         if (v)
490                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
491                                 " count=%lu.\n", cpu, v);
492
493         v = cb->loss_of_sample_data;
494         if (v)
495                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
496                                 " count=%lu.\n", cpu, v);
497
498         v = cb->invalid_entry_address;
499         if (v)
500                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
501                                 " count=%lu.\n", cpu, v);
502
503         v = cb->incorrect_sdbt_entry;
504         if (v)
505                 printk(KERN_ERR
506                                 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
507                                 " count=%lu.\n", cpu, v);
508
509         v = cb->sample_auth_change_alert;
510         if (v)
511                 printk(KERN_ERR
512                                 "hwsampler: CPU %d CPUMF Sample authorization change,"
513                                 " count=%lu.\n", cpu, v);
514
515         return rc;
516 }
517
518 static int check_hardware_prerequisites(void)
519 {
520         if (!test_facility(68))
521                 return -EOPNOTSUPP;
522         return 0;
523 }
524 /*
525  * hws_oom_callback() - the OOM callback function
526  *
527  * In case the callback is invoked during memory allocation for the
528  *  hw sampler, all obtained memory is deallocated and a flag is set
529  *  so main sampler memory allocation can exit with a failure code.
530  * In case the callback is invoked during sampling the hw sampler
531  *  is deactivated for all CPUs.
532  */
533 static int hws_oom_callback(struct notifier_block *nfb,
534         unsigned long dummy, void *parm)
535 {
536         unsigned long *freed;
537         int cpu;
538         struct hws_cpu_buffer *cb;
539
540         freed = parm;
541
542         mutex_lock(&hws_sem_oom);
543
544         if (hws_state == HWS_DEALLOCATED) {
545                 /* during memory allocation */
546                 if (oom_killer_was_active == 0) {
547                         oom_killer_was_active = 1;
548                         *freed += deallocate_sdbt();
549                 }
550         } else {
551                 int i;
552                 cpu = get_cpu();
553                 cb = &per_cpu(sampler_cpu_buffer, cpu);
554
555                 if (!cb->oom) {
556                         for_each_online_cpu(i) {
557                                 smp_ctl_ssctl_deactivate(i);
558                                 cb->oom = 1;
559                         }
560                         cb->finish = 1;
561
562                         printk(KERN_INFO
563                                 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
564                                 cpu);
565                 }
566         }
567
568         mutex_unlock(&hws_sem_oom);
569
570         return NOTIFY_OK;
571 }
572
573 static struct notifier_block hws_oom_notifier = {
574         .notifier_call = hws_oom_callback
575 };
576
577 static int hws_cpu_callback(struct notifier_block *nfb,
578         unsigned long action, void *hcpu)
579 {
580         /* We do not have sampler space available for all possible CPUs.
581            All CPUs should be online when hw sampling is activated. */
582         return NOTIFY_BAD;
583 }
584
585 static struct notifier_block hws_cpu_notifier = {
586         .notifier_call = hws_cpu_callback
587 };
588
589 /**
590  * hwsampler_deactivate() - set hardware sampling temporarily inactive
591  * @cpu:  specifies the CPU to be set inactive.
592  *
593  * Returns 0 on success, !0 on failure.
594  */
595 int hwsampler_deactivate(unsigned int cpu)
596 {
597         /*
598          * Deactivate hw sampling temporarily and flush the buffer
599          * by pushing all the pending samples to oprofile buffer.
600          *
601          * This function can be called under one of the following conditions:
602          *     Memory unmap, task is exiting.
603          */
604         int rc;
605         struct hws_cpu_buffer *cb;
606
607         rc = 0;
608         mutex_lock(&hws_sem);
609
610         cb = &per_cpu(sampler_cpu_buffer, cpu);
611         if (hws_state == HWS_STARTED) {
612                 rc = smp_ctl_qsi(cpu);
613                 WARN_ON(rc);
614                 if (cb->qsi.cs) {
615                         rc = smp_ctl_ssctl_deactivate(cpu);
616                         if (rc) {
617                                 printk(KERN_INFO
618                                 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
619                                 cb->finish = 1;
620                                 hws_state = HWS_STOPPING;
621                         } else  {
622                                 hws_flush_all = 1;
623                                 /* Add work to queue to read pending samples.*/
624                                 queue_work_on(cpu, hws_wq, &cb->worker);
625                         }
626                 }
627         }
628         mutex_unlock(&hws_sem);
629
630         if (hws_wq)
631                 flush_workqueue(hws_wq);
632
633         return rc;
634 }
635
636 /**
637  * hwsampler_activate() - activate/resume hardware sampling which was deactivated
638  * @cpu:  specifies the CPU to be set active.
639  *
640  * Returns 0 on success, !0 on failure.
641  */
642 int hwsampler_activate(unsigned int cpu)
643 {
644         /*
645          * Re-activate hw sampling. This should be called in pair with
646          * hwsampler_deactivate().
647          */
648         int rc;
649         struct hws_cpu_buffer *cb;
650
651         rc = 0;
652         mutex_lock(&hws_sem);
653
654         cb = &per_cpu(sampler_cpu_buffer, cpu);
655         if (hws_state == HWS_STARTED) {
656                 rc = smp_ctl_qsi(cpu);
657                 WARN_ON(rc);
658                 if (!cb->qsi.cs) {
659                         hws_flush_all = 0;
660                         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
661                         if (rc) {
662                                 printk(KERN_ERR
663                                 "CPU %d, CPUMF activate sampling failed.\n",
664                                          cpu);
665                         }
666                 }
667         }
668
669         mutex_unlock(&hws_sem);
670
671         return rc;
672 }
673
674 static void hws_ext_handler(unsigned int ext_int_code,
675                             unsigned int param32, unsigned long param64)
676 {
677         int cpu;
678         struct hws_cpu_buffer *cb;
679
680         cpu = smp_processor_id();
681         cb = &per_cpu(sampler_cpu_buffer, cpu);
682
683         atomic_xchg(
684                         &cb->ext_params,
685                         atomic_read(&cb->ext_params)
686                                 | S390_lowcore.ext_params);
687
688         if (hws_wq)
689                 queue_work(hws_wq, &cb->worker);
690 }
691
692 static int check_qsi_on_setup(void)
693 {
694         int rc;
695         unsigned int cpu;
696         struct hws_cpu_buffer *cb;
697
698         for_each_online_cpu(cpu) {
699                 cb = &per_cpu(sampler_cpu_buffer, cpu);
700                 rc = smp_ctl_qsi(cpu);
701                 WARN_ON(rc);
702                 if (rc)
703                         return -EOPNOTSUPP;
704
705                 if (!cb->qsi.as) {
706                         printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
707                         return -EINVAL;
708                 }
709
710                 if (cb->qsi.es) {
711                         printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
712                         rc = smp_ctl_ssctl_stop(cpu);
713                         if (rc)
714                                 return -EINVAL;
715
716                         printk(KERN_INFO
717                                 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
718                 }
719         }
720         return 0;
721 }
722
723 static int check_qsi_on_start(void)
724 {
725         unsigned int cpu;
726         int rc;
727         struct hws_cpu_buffer *cb;
728
729         for_each_online_cpu(cpu) {
730                 cb = &per_cpu(sampler_cpu_buffer, cpu);
731                 rc = smp_ctl_qsi(cpu);
732                 WARN_ON(rc);
733
734                 if (!cb->qsi.as)
735                         return -EINVAL;
736
737                 if (cb->qsi.es)
738                         return -EINVAL;
739
740                 if (cb->qsi.cs)
741                         return -EINVAL;
742         }
743         return 0;
744 }
745
746 static void worker_on_start(unsigned int cpu)
747 {
748         struct hws_cpu_buffer *cb;
749
750         cb = &per_cpu(sampler_cpu_buffer, cpu);
751         cb->worker_entry = cb->first_sdbt;
752 }
753
754 static int worker_check_error(unsigned int cpu, int ext_params)
755 {
756         int rc;
757         unsigned long *sdbt;
758         struct hws_cpu_buffer *cb;
759
760         rc = 0;
761         cb = &per_cpu(sampler_cpu_buffer, cpu);
762         sdbt = (unsigned long *) cb->worker_entry;
763
764         if (!sdbt || !*sdbt)
765                 return -EINVAL;
766
767         if (ext_params & EI_IEA)
768                 cb->req_alert++;
769
770         if (ext_params & EI_LSDA)
771                 cb->loss_of_sample_data++;
772
773         if (ext_params & EI_IEA) {
774                 cb->invalid_entry_address++;
775                 rc = -EINVAL;
776         }
777
778         if (ext_params & EI_ISE) {
779                 cb->incorrect_sdbt_entry++;
780                 rc = -EINVAL;
781         }
782
783         if (ext_params & EI_SACA) {
784                 cb->sample_auth_change_alert++;
785                 rc = -EINVAL;
786         }
787
788         return rc;
789 }
790
791 static void worker_on_finish(unsigned int cpu)
792 {
793         int rc, i;
794         struct hws_cpu_buffer *cb;
795
796         cb = &per_cpu(sampler_cpu_buffer, cpu);
797
798         if (cb->finish) {
799                 rc = smp_ctl_qsi(cpu);
800                 WARN_ON(rc);
801                 if (cb->qsi.es) {
802                         printk(KERN_INFO
803                                 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
804                                 cpu);
805                         rc = smp_ctl_ssctl_stop(cpu);
806                         if (rc)
807                                 printk(KERN_INFO
808                                         "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
809                                         cpu);
810
811                         for_each_online_cpu(i) {
812                                 if (i == cpu)
813                                         continue;
814                                 if (!cb->finish) {
815                                         cb->finish = 1;
816                                         queue_work_on(i, hws_wq,
817                                                 &cb->worker);
818                                 }
819                         }
820                 }
821         }
822 }
823
824 static void worker_on_interrupt(unsigned int cpu)
825 {
826         unsigned long *sdbt;
827         unsigned char done;
828         struct hws_cpu_buffer *cb;
829
830         cb = &per_cpu(sampler_cpu_buffer, cpu);
831
832         sdbt = (unsigned long *) cb->worker_entry;
833
834         done = 0;
835         /* do not proceed if stop was entered,
836          * forget the buffers not yet processed */
837         while (!done && !cb->stop_mode) {
838                 unsigned long *trailer;
839                 struct hws_trailer_entry *te;
840                 unsigned long *dear = 0;
841
842                 trailer = trailer_entry_ptr(*sdbt);
843                 /* leave loop if no more work to do */
844                 if (!(*trailer & BUFFER_FULL_MASK)) {
845                         done = 1;
846                         if (!hws_flush_all)
847                                 continue;
848                 }
849
850                 te = (struct hws_trailer_entry *)trailer;
851                 cb->sample_overflow += te->overflow;
852
853                 add_samples_to_oprofile(cpu, sdbt, dear);
854
855                 /* reset trailer */
856                 xchg((unsigned char *) te, 0x40);
857
858                 /* advance to next sdb slot in current sdbt */
859                 sdbt++;
860                 /* in case link bit is set use address w/o link bit */
861                 if (is_link_entry(sdbt))
862                         sdbt = get_next_sdbt(sdbt);
863
864                 cb->worker_entry = (unsigned long)sdbt;
865         }
866 }
867
868 static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
869                 unsigned long *dear)
870 {
871         struct hws_data_entry *sample_data_ptr;
872         unsigned long *trailer;
873
874         trailer = trailer_entry_ptr(*sdbt);
875         if (dear) {
876                 if (dear > trailer)
877                         return;
878                 trailer = dear;
879         }
880
881         sample_data_ptr = (struct hws_data_entry *)(*sdbt);
882
883         while ((unsigned long *)sample_data_ptr < trailer) {
884                 struct pt_regs *regs = NULL;
885                 struct task_struct *tsk = NULL;
886
887                 /*
888                  * Check sampling mode, 1 indicates basic (=customer) sampling
889                  * mode.
890                  */
891                 if (sample_data_ptr->def != 1) {
892                         /* sample slot is not yet written */
893                         break;
894                 } else {
895                         /* make sure we don't use it twice,
896                          * the next time the sampler will set it again */
897                         sample_data_ptr->def = 0;
898                 }
899
900                 /* Get pt_regs. */
901                 if (sample_data_ptr->P == 1) {
902                         /* userspace sample */
903                         unsigned int pid = sample_data_ptr->prim_asn;
904                         rcu_read_lock();
905                         tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
906                         if (tsk)
907                                 regs = task_pt_regs(tsk);
908                         rcu_read_unlock();
909                 } else {
910                         /* kernelspace sample */
911                         regs = task_pt_regs(current);
912                 }
913
914                 mutex_lock(&hws_sem);
915                 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
916                                 !sample_data_ptr->P, tsk);
917                 mutex_unlock(&hws_sem);
918
919                 sample_data_ptr++;
920         }
921 }
922
923 static void worker(struct work_struct *work)
924 {
925         unsigned int cpu;
926         int ext_params;
927         struct hws_cpu_buffer *cb;
928
929         cb = container_of(work, struct hws_cpu_buffer, worker);
930         cpu = smp_processor_id();
931         ext_params = atomic_xchg(&cb->ext_params, 0);
932
933         if (!cb->worker_entry)
934                 worker_on_start(cpu);
935
936         if (worker_check_error(cpu, ext_params))
937                 return;
938
939         if (!cb->finish)
940                 worker_on_interrupt(cpu);
941
942         if (cb->finish)
943                 worker_on_finish(cpu);
944 }
945
946 /**
947  * hwsampler_allocate() - allocate memory for the hardware sampler
948  * @sdbt:  number of SDBTs per online CPU (must be > 0)
949  * @sdb:   number of SDBs per SDBT (minimum 1, maximum 511)
950  *
951  * Returns 0 on success, !0 on failure.
952  */
953 int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
954 {
955         int cpu, rc;
956         mutex_lock(&hws_sem);
957
958         rc = -EINVAL;
959         if (hws_state != HWS_DEALLOCATED)
960                 goto allocate_exit;
961
962         if (sdbt < 1)
963                 goto allocate_exit;
964
965         if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
966                 goto allocate_exit;
967
968         num_sdbt = sdbt;
969         num_sdb = sdb;
970
971         oom_killer_was_active = 0;
972         register_oom_notifier(&hws_oom_notifier);
973
974         for_each_online_cpu(cpu) {
975                 if (allocate_sdbt(cpu)) {
976                         unregister_oom_notifier(&hws_oom_notifier);
977                         goto allocate_error;
978                 }
979         }
980         unregister_oom_notifier(&hws_oom_notifier);
981         if (oom_killer_was_active)
982                 goto allocate_error;
983
984         hws_state = HWS_STOPPED;
985         rc = 0;
986
987 allocate_exit:
988         mutex_unlock(&hws_sem);
989         return rc;
990
991 allocate_error:
992         rc = -ENOMEM;
993         printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
994         goto allocate_exit;
995 }
996
997 /**
998  * hwsampler_deallocate() - deallocate hardware sampler memory
999  *
1000  * Returns 0 on success, !0 on failure.
1001  */
1002 int hwsampler_deallocate()
1003 {
1004         int rc;
1005
1006         mutex_lock(&hws_sem);
1007
1008         rc = -EINVAL;
1009         if (hws_state != HWS_STOPPED)
1010                 goto deallocate_exit;
1011
1012         smp_ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1013         deallocate_sdbt();
1014
1015         hws_state = HWS_DEALLOCATED;
1016         rc = 0;
1017
1018 deallocate_exit:
1019         mutex_unlock(&hws_sem);
1020
1021         return rc;
1022 }
1023
1024 unsigned long hwsampler_query_min_interval(void)
1025 {
1026         return min_sampler_rate;
1027 }
1028
1029 unsigned long hwsampler_query_max_interval(void)
1030 {
1031         return max_sampler_rate;
1032 }
1033
1034 unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
1035 {
1036         struct hws_cpu_buffer *cb;
1037
1038         cb = &per_cpu(sampler_cpu_buffer, cpu);
1039
1040         return cb->sample_overflow;
1041 }
1042
1043 int hwsampler_setup()
1044 {
1045         int rc;
1046         int cpu;
1047         struct hws_cpu_buffer *cb;
1048
1049         mutex_lock(&hws_sem);
1050
1051         rc = -EINVAL;
1052         if (hws_state)
1053                 goto setup_exit;
1054
1055         hws_state = HWS_INIT;
1056
1057         init_all_cpu_buffers();
1058
1059         rc = check_hardware_prerequisites();
1060         if (rc)
1061                 goto setup_exit;
1062
1063         rc = check_qsi_on_setup();
1064         if (rc)
1065                 goto setup_exit;
1066
1067         rc = -EINVAL;
1068         hws_wq = create_workqueue("hwsampler");
1069         if (!hws_wq)
1070                 goto setup_exit;
1071
1072         register_cpu_notifier(&hws_cpu_notifier);
1073
1074         for_each_online_cpu(cpu) {
1075                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1076                 INIT_WORK(&cb->worker, worker);
1077                 rc = smp_ctl_qsi(cpu);
1078                 WARN_ON(rc);
1079                 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1080                         if (min_sampler_rate) {
1081                                 printk(KERN_WARNING
1082                                         "hwsampler: different min sampler rate values.\n");
1083                                 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1084                                         min_sampler_rate =
1085                                                 cb->qsi.min_sampl_rate;
1086                         } else
1087                                 min_sampler_rate = cb->qsi.min_sampl_rate;
1088                 }
1089                 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1090                         if (max_sampler_rate) {
1091                                 printk(KERN_WARNING
1092                                         "hwsampler: different max sampler rate values.\n");
1093                                 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1094                                         max_sampler_rate =
1095                                                 cb->qsi.max_sampl_rate;
1096                         } else
1097                                 max_sampler_rate = cb->qsi.max_sampl_rate;
1098                 }
1099         }
1100         register_external_interrupt(0x1407, hws_ext_handler);
1101
1102         hws_state = HWS_DEALLOCATED;
1103         rc = 0;
1104
1105 setup_exit:
1106         mutex_unlock(&hws_sem);
1107         return rc;
1108 }
1109
1110 int hwsampler_shutdown()
1111 {
1112         int rc;
1113
1114         mutex_lock(&hws_sem);
1115
1116         rc = -EINVAL;
1117         if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1118                 mutex_unlock(&hws_sem);
1119
1120                 if (hws_wq)
1121                         flush_workqueue(hws_wq);
1122
1123                 mutex_lock(&hws_sem);
1124
1125                 if (hws_state == HWS_STOPPED) {
1126                         smp_ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1127                         deallocate_sdbt();
1128                 }
1129                 if (hws_wq) {
1130                         destroy_workqueue(hws_wq);
1131                         hws_wq = NULL;
1132                 }
1133
1134                 unregister_external_interrupt(0x1407, hws_ext_handler);
1135                 hws_state = HWS_INIT;
1136                 rc = 0;
1137         }
1138         mutex_unlock(&hws_sem);
1139
1140         unregister_cpu_notifier(&hws_cpu_notifier);
1141
1142         return rc;
1143 }
1144
1145 /**
1146  * hwsampler_start_all() - start hardware sampling on all online CPUs
1147  * @rate:  specifies the used interval when samples are taken
1148  *
1149  * Returns 0 on success, !0 on failure.
1150  */
1151 int hwsampler_start_all(unsigned long rate)
1152 {
1153         int rc, cpu;
1154
1155         mutex_lock(&hws_sem);
1156
1157         hws_oom = 0;
1158
1159         rc = -EINVAL;
1160         if (hws_state != HWS_STOPPED)
1161                 goto start_all_exit;
1162
1163         interval = rate;
1164
1165         /* fail if rate is not valid */
1166         if (interval < min_sampler_rate || interval > max_sampler_rate)
1167                 goto start_all_exit;
1168
1169         rc = check_qsi_on_start();
1170         if (rc)
1171                 goto start_all_exit;
1172
1173         rc = prepare_cpu_buffers();
1174         if (rc)
1175                 goto start_all_exit;
1176
1177         for_each_online_cpu(cpu) {
1178                 rc = start_sampling(cpu);
1179                 if (rc)
1180                         break;
1181         }
1182         if (rc) {
1183                 for_each_online_cpu(cpu) {
1184                         stop_sampling(cpu);
1185                 }
1186                 goto start_all_exit;
1187         }
1188         hws_state = HWS_STARTED;
1189         rc = 0;
1190
1191 start_all_exit:
1192         mutex_unlock(&hws_sem);
1193
1194         if (rc)
1195                 return rc;
1196
1197         register_oom_notifier(&hws_oom_notifier);
1198         hws_oom = 1;
1199         hws_flush_all = 0;
1200         /* now let them in, 1407 CPUMF external interrupts */
1201         smp_ctl_set_bit(0, 5); /* set CR0 bit 58 */
1202
1203         return 0;
1204 }
1205
1206 /**
1207  * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1208  *
1209  * Returns 0 on success, !0 on failure.
1210  */
1211 int hwsampler_stop_all()
1212 {
1213         int tmp_rc, rc, cpu;
1214         struct hws_cpu_buffer *cb;
1215
1216         mutex_lock(&hws_sem);
1217
1218         rc = 0;
1219         if (hws_state == HWS_INIT) {
1220                 mutex_unlock(&hws_sem);
1221                 return rc;
1222         }
1223         hws_state = HWS_STOPPING;
1224         mutex_unlock(&hws_sem);
1225
1226         for_each_online_cpu(cpu) {
1227                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1228                 cb->stop_mode = 1;
1229                 tmp_rc = stop_sampling(cpu);
1230                 if (tmp_rc)
1231                         rc = tmp_rc;
1232         }
1233
1234         if (hws_wq)
1235                 flush_workqueue(hws_wq);
1236
1237         mutex_lock(&hws_sem);
1238         if (hws_oom) {
1239                 unregister_oom_notifier(&hws_oom_notifier);
1240                 hws_oom = 0;
1241         }
1242         hws_state = HWS_STOPPED;
1243         mutex_unlock(&hws_sem);
1244
1245         return rc;
1246 }