include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit...
[linux-2.6.git] / arch / powerpc / oprofile / cell / spu_task_sync.c
1 /*
2  * Cell Broadband Engine OProfile Support
3  *
4  * (C) Copyright IBM Corporation 2006
5  *
6  * Author: Maynard Johnson <maynardj@us.ibm.com>
7  *
8  * This program is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU General Public License
10  * as published by the Free Software Foundation; either version
11  * 2 of the License, or (at your option) any later version.
12  */
13
14 /* The purpose of this file is to handle SPU event task switching
15  * and to record SPU context information into the OProfile
16  * event buffer.
17  *
18  * Additionally, the spu_sync_buffer function is provided as a helper
19  * for recoding actual SPU program counter samples to the event buffer.
20  */
21 #include <linux/dcookies.h>
22 #include <linux/kref.h>
23 #include <linux/mm.h>
24 #include <linux/fs.h>
25 #include <linux/module.h>
26 #include <linux/notifier.h>
27 #include <linux/numa.h>
28 #include <linux/oprofile.h>
29 #include <linux/slab.h>
30 #include <linux/spinlock.h>
31 #include "pr_util.h"
32
33 #define RELEASE_ALL 9999
34
35 static DEFINE_SPINLOCK(buffer_lock);
36 static DEFINE_SPINLOCK(cache_lock);
37 static int num_spu_nodes;
38 int spu_prof_num_nodes;
39
40 struct spu_buffer spu_buff[MAX_NUMNODES * SPUS_PER_NODE];
41 struct delayed_work spu_work;
42 static unsigned max_spu_buff;
43
44 static void spu_buff_add(unsigned long int value, int spu)
45 {
46         /* spu buff is a circular buffer.  Add entries to the
47          * head.  Head is the index to store the next value.
48          * The buffer is full when there is one available entry
49          * in the queue, i.e. head and tail can't be equal.
50          * That way we can tell the difference between the
51          * buffer being full versus empty.
52          *
53          *  ASSUPTION: the buffer_lock is held when this function
54          *             is called to lock the buffer, head and tail.
55          */
56         int full = 1;
57
58         if (spu_buff[spu].head >= spu_buff[spu].tail) {
59                 if ((spu_buff[spu].head - spu_buff[spu].tail)
60                     <  (max_spu_buff - 1))
61                         full = 0;
62
63         } else if (spu_buff[spu].tail > spu_buff[spu].head) {
64                 if ((spu_buff[spu].tail - spu_buff[spu].head)
65                     > 1)
66                         full = 0;
67         }
68
69         if (!full) {
70                 spu_buff[spu].buff[spu_buff[spu].head] = value;
71                 spu_buff[spu].head++;
72
73                 if (spu_buff[spu].head >= max_spu_buff)
74                         spu_buff[spu].head = 0;
75         } else {
76                 /* From the user's perspective make the SPU buffer
77                  * size management/overflow look like we are using
78                  * per cpu buffers.  The user uses the same
79                  * per cpu parameter to adjust the SPU buffer size.
80                  * Increment the sample_lost_overflow to inform
81                  * the user the buffer size needs to be increased.
82                  */
83                 oprofile_cpu_buffer_inc_smpl_lost();
84         }
85 }
86
87 /* This function copies the per SPU buffers to the
88  * OProfile kernel buffer.
89  */
90 void sync_spu_buff(void)
91 {
92         int spu;
93         unsigned long flags;
94         int curr_head;
95
96         for (spu = 0; spu < num_spu_nodes; spu++) {
97                 /* In case there was an issue and the buffer didn't
98                  * get created skip it.
99                  */
100                 if (spu_buff[spu].buff == NULL)
101                         continue;
102
103                 /* Hold the lock to make sure the head/tail
104                  * doesn't change while spu_buff_add() is
105                  * deciding if the buffer is full or not.
106                  * Being a little paranoid.
107                  */
108                 spin_lock_irqsave(&buffer_lock, flags);
109                 curr_head = spu_buff[spu].head;
110                 spin_unlock_irqrestore(&buffer_lock, flags);
111
112                 /* Transfer the current contents to the kernel buffer.
113                  * data can still be added to the head of the buffer.
114                  */
115                 oprofile_put_buff(spu_buff[spu].buff,
116                                   spu_buff[spu].tail,
117                                   curr_head, max_spu_buff);
118
119                 spin_lock_irqsave(&buffer_lock, flags);
120                 spu_buff[spu].tail = curr_head;
121                 spin_unlock_irqrestore(&buffer_lock, flags);
122         }
123
124 }
125
126 static void wq_sync_spu_buff(struct work_struct *work)
127 {
128         /* move data from spu buffers to kernel buffer */
129         sync_spu_buff();
130
131         /* only reschedule if profiling is not done */
132         if (spu_prof_running)
133                 schedule_delayed_work(&spu_work, DEFAULT_TIMER_EXPIRE);
134 }
135
136 /* Container for caching information about an active SPU task. */
137 struct cached_info {
138         struct vma_to_fileoffset_map *map;
139         struct spu *the_spu;    /* needed to access pointer to local_store */
140         struct kref cache_ref;
141 };
142
143 static struct cached_info *spu_info[MAX_NUMNODES * 8];
144
145 static void destroy_cached_info(struct kref *kref)
146 {
147         struct cached_info *info;
148
149         info = container_of(kref, struct cached_info, cache_ref);
150         vma_map_free(info->map);
151         kfree(info);
152         module_put(THIS_MODULE);
153 }
154
155 /* Return the cached_info for the passed SPU number.
156  * ATTENTION:  Callers are responsible for obtaining the
157  *             cache_lock if needed prior to invoking this function.
158  */
159 static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num)
160 {
161         struct kref *ref;
162         struct cached_info *ret_info;
163
164         if (spu_num >= num_spu_nodes) {
165                 printk(KERN_ERR "SPU_PROF: "
166                        "%s, line %d: Invalid index %d into spu info cache\n",
167                        __func__, __LINE__, spu_num);
168                 ret_info = NULL;
169                 goto out;
170         }
171         if (!spu_info[spu_num] && the_spu) {
172                 ref = spu_get_profile_private_kref(the_spu->ctx);
173                 if (ref) {
174                         spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref);
175                         kref_get(&spu_info[spu_num]->cache_ref);
176                 }
177         }
178
179         ret_info = spu_info[spu_num];
180  out:
181         return ret_info;
182 }
183
184
185 /* Looks for cached info for the passed spu.  If not found, the
186  * cached info is created for the passed spu.
187  * Returns 0 for success; otherwise, -1 for error.
188  */
189 static int
190 prepare_cached_spu_info(struct spu *spu, unsigned long objectId)
191 {
192         unsigned long flags;
193         struct vma_to_fileoffset_map *new_map;
194         int retval = 0;
195         struct cached_info *info;
196
197         /* We won't bother getting cache_lock here since
198          * don't do anything with the cached_info that's returned.
199          */
200         info = get_cached_info(spu, spu->number);
201
202         if (info) {
203                 pr_debug("Found cached SPU info.\n");
204                 goto out;
205         }
206
207         /* Create cached_info and set spu_info[spu->number] to point to it.
208          * spu->number is a system-wide value, not a per-node value.
209          */
210         info = kzalloc(sizeof(struct cached_info), GFP_KERNEL);
211         if (!info) {
212                 printk(KERN_ERR "SPU_PROF: "
213                        "%s, line %d: create vma_map failed\n",
214                        __func__, __LINE__);
215                 retval = -ENOMEM;
216                 goto err_alloc;
217         }
218         new_map = create_vma_map(spu, objectId);
219         if (!new_map) {
220                 printk(KERN_ERR "SPU_PROF: "
221                        "%s, line %d: create vma_map failed\n",
222                        __func__, __LINE__);
223                 retval = -ENOMEM;
224                 goto err_alloc;
225         }
226
227         pr_debug("Created vma_map\n");
228         info->map = new_map;
229         info->the_spu = spu;
230         kref_init(&info->cache_ref);
231         spin_lock_irqsave(&cache_lock, flags);
232         spu_info[spu->number] = info;
233         /* Increment count before passing off ref to SPUFS. */
234         kref_get(&info->cache_ref);
235
236         /* We increment the module refcount here since SPUFS is
237          * responsible for the final destruction of the cached_info,
238          * and it must be able to access the destroy_cached_info()
239          * function defined in the OProfile module.  We decrement
240          * the module refcount in destroy_cached_info.
241          */
242         try_module_get(THIS_MODULE);
243         spu_set_profile_private_kref(spu->ctx, &info->cache_ref,
244                                 destroy_cached_info);
245         spin_unlock_irqrestore(&cache_lock, flags);
246         goto out;
247
248 err_alloc:
249         kfree(info);
250 out:
251         return retval;
252 }
253
254 /*
255  * NOTE:  The caller is responsible for locking the
256  *        cache_lock prior to calling this function.
257  */
258 static int release_cached_info(int spu_index)
259 {
260         int index, end;
261
262         if (spu_index == RELEASE_ALL) {
263                 end = num_spu_nodes;
264                 index = 0;
265         } else {
266                 if (spu_index >= num_spu_nodes) {
267                         printk(KERN_ERR "SPU_PROF: "
268                                 "%s, line %d: "
269                                 "Invalid index %d into spu info cache\n",
270                                 __func__, __LINE__, spu_index);
271                         goto out;
272                 }
273                 end = spu_index + 1;
274                 index = spu_index;
275         }
276         for (; index < end; index++) {
277                 if (spu_info[index]) {
278                         kref_put(&spu_info[index]->cache_ref,
279                                  destroy_cached_info);
280                         spu_info[index] = NULL;
281                 }
282         }
283
284 out:
285         return 0;
286 }
287
288 /* The source code for fast_get_dcookie was "borrowed"
289  * from drivers/oprofile/buffer_sync.c.
290  */
291
292 /* Optimisation. We can manage without taking the dcookie sem
293  * because we cannot reach this code without at least one
294  * dcookie user still being registered (namely, the reader
295  * of the event buffer).
296  */
297 static inline unsigned long fast_get_dcookie(struct path *path)
298 {
299         unsigned long cookie;
300
301         if (path->dentry->d_flags & DCACHE_COOKIE)
302                 return (unsigned long)path->dentry;
303         get_dcookie(path, &cookie);
304         return cookie;
305 }
306
307 /* Look up the dcookie for the task's first VM_EXECUTABLE mapping,
308  * which corresponds loosely to "application name". Also, determine
309  * the offset for the SPU ELF object.  If computed offset is
310  * non-zero, it implies an embedded SPU object; otherwise, it's a
311  * separate SPU binary, in which case we retrieve it's dcookie.
312  * For the embedded case, we must determine if SPU ELF is embedded
313  * in the executable application or another file (i.e., shared lib).
314  * If embedded in a shared lib, we must get the dcookie and return
315  * that to the caller.
316  */
317 static unsigned long
318 get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp,
319                             unsigned long *spu_bin_dcookie,
320                             unsigned long spu_ref)
321 {
322         unsigned long app_cookie = 0;
323         unsigned int my_offset = 0;
324         struct file *app = NULL;
325         struct vm_area_struct *vma;
326         struct mm_struct *mm = spu->mm;
327
328         if (!mm)
329                 goto out;
330
331         down_read(&mm->mmap_sem);
332
333         for (vma = mm->mmap; vma; vma = vma->vm_next) {
334                 if (!vma->vm_file)
335                         continue;
336                 if (!(vma->vm_flags & VM_EXECUTABLE))
337                         continue;
338                 app_cookie = fast_get_dcookie(&vma->vm_file->f_path);
339                 pr_debug("got dcookie for %s\n",
340                          vma->vm_file->f_dentry->d_name.name);
341                 app = vma->vm_file;
342                 break;
343         }
344
345         for (vma = mm->mmap; vma; vma = vma->vm_next) {
346                 if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref)
347                         continue;
348                 my_offset = spu_ref - vma->vm_start;
349                 if (!vma->vm_file)
350                         goto fail_no_image_cookie;
351
352                 pr_debug("Found spu ELF at %X(object-id:%lx) for file %s\n",
353                          my_offset, spu_ref,
354                          vma->vm_file->f_dentry->d_name.name);
355                 *offsetp = my_offset;
356                 break;
357         }
358
359         *spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path);
360         pr_debug("got dcookie for %s\n", vma->vm_file->f_dentry->d_name.name);
361
362         up_read(&mm->mmap_sem);
363
364 out:
365         return app_cookie;
366
367 fail_no_image_cookie:
368         up_read(&mm->mmap_sem);
369
370         printk(KERN_ERR "SPU_PROF: "
371                 "%s, line %d: Cannot find dcookie for SPU binary\n",
372                 __func__, __LINE__);
373         goto out;
374 }
375
376
377
378 /* This function finds or creates cached context information for the
379  * passed SPU and records SPU context information into the OProfile
380  * event buffer.
381  */
382 static int process_context_switch(struct spu *spu, unsigned long objectId)
383 {
384         unsigned long flags;
385         int retval;
386         unsigned int offset = 0;
387         unsigned long spu_cookie = 0, app_dcookie;
388
389         retval = prepare_cached_spu_info(spu, objectId);
390         if (retval)
391                 goto out;
392
393         /* Get dcookie first because a mutex_lock is taken in that
394          * code path, so interrupts must not be disabled.
395          */
396         app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId);
397         if (!app_dcookie || !spu_cookie) {
398                 retval  = -ENOENT;
399                 goto out;
400         }
401
402         /* Record context info in event buffer */
403         spin_lock_irqsave(&buffer_lock, flags);
404         spu_buff_add(ESCAPE_CODE, spu->number);
405         spu_buff_add(SPU_CTX_SWITCH_CODE, spu->number);
406         spu_buff_add(spu->number, spu->number);
407         spu_buff_add(spu->pid, spu->number);
408         spu_buff_add(spu->tgid, spu->number);
409         spu_buff_add(app_dcookie, spu->number);
410         spu_buff_add(spu_cookie, spu->number);
411         spu_buff_add(offset, spu->number);
412
413         /* Set flag to indicate SPU PC data can now be written out.  If
414          * the SPU program counter data is seen before an SPU context
415          * record is seen, the postprocessing will fail.
416          */
417         spu_buff[spu->number].ctx_sw_seen = 1;
418
419         spin_unlock_irqrestore(&buffer_lock, flags);
420         smp_wmb();      /* insure spu event buffer updates are written */
421                         /* don't want entries intermingled... */
422 out:
423         return retval;
424 }
425
426 /*
427  * This function is invoked on either a bind_context or unbind_context.
428  * If called for an unbind_context, the val arg is 0; otherwise,
429  * it is the object-id value for the spu context.
430  * The data arg is of type 'struct spu *'.
431  */
432 static int spu_active_notify(struct notifier_block *self, unsigned long val,
433                                 void *data)
434 {
435         int retval;
436         unsigned long flags;
437         struct spu *the_spu = data;
438
439         pr_debug("SPU event notification arrived\n");
440         if (!val) {
441                 spin_lock_irqsave(&cache_lock, flags);
442                 retval = release_cached_info(the_spu->number);
443                 spin_unlock_irqrestore(&cache_lock, flags);
444         } else {
445                 retval = process_context_switch(the_spu, val);
446         }
447         return retval;
448 }
449
450 static struct notifier_block spu_active = {
451         .notifier_call = spu_active_notify,
452 };
453
454 static int number_of_online_nodes(void)
455 {
456         u32 cpu; u32 tmp;
457         int nodes = 0;
458         for_each_online_cpu(cpu) {
459                 tmp = cbe_cpu_to_node(cpu) + 1;
460                 if (tmp > nodes)
461                         nodes++;
462         }
463         return nodes;
464 }
465
466 static int oprofile_spu_buff_create(void)
467 {
468         int spu;
469
470         max_spu_buff = oprofile_get_cpu_buffer_size();
471
472         for (spu = 0; spu < num_spu_nodes; spu++) {
473                 /* create circular buffers to store the data in.
474                  * use locks to manage accessing the buffers
475                  */
476                 spu_buff[spu].head = 0;
477                 spu_buff[spu].tail = 0;
478
479                 /*
480                  * Create a buffer for each SPU.  Can't reliably
481                  * create a single buffer for all spus due to not
482                  * enough contiguous kernel memory.
483                  */
484
485                 spu_buff[spu].buff = kzalloc((max_spu_buff
486                                               * sizeof(unsigned long)),
487                                              GFP_KERNEL);
488
489                 if (!spu_buff[spu].buff) {
490                         printk(KERN_ERR "SPU_PROF: "
491                                "%s, line %d:  oprofile_spu_buff_create "
492                        "failed to allocate spu buffer %d.\n",
493                                __func__, __LINE__, spu);
494
495                         /* release the spu buffers that have been allocated */
496                         while (spu >= 0) {
497                                 kfree(spu_buff[spu].buff);
498                                 spu_buff[spu].buff = 0;
499                                 spu--;
500                         }
501                         return -ENOMEM;
502                 }
503         }
504         return 0;
505 }
506
507 /* The main purpose of this function is to synchronize
508  * OProfile with SPUFS by registering to be notified of
509  * SPU task switches.
510  *
511  * NOTE: When profiling SPUs, we must ensure that only
512  * spu_sync_start is invoked and not the generic sync_start
513  * in drivers/oprofile/oprof.c.  A return value of
514  * SKIP_GENERIC_SYNC or SYNC_START_ERROR will
515  * accomplish this.
516  */
517 int spu_sync_start(void)
518 {
519         int spu;
520         int ret = SKIP_GENERIC_SYNC;
521         int register_ret;
522         unsigned long flags = 0;
523
524         spu_prof_num_nodes = number_of_online_nodes();
525         num_spu_nodes = spu_prof_num_nodes * 8;
526         INIT_DELAYED_WORK(&spu_work, wq_sync_spu_buff);
527
528         /* create buffer for storing the SPU data to put in
529          * the kernel buffer.
530          */
531         ret = oprofile_spu_buff_create();
532         if (ret)
533                 goto out;
534
535         spin_lock_irqsave(&buffer_lock, flags);
536         for (spu = 0; spu < num_spu_nodes; spu++) {
537                 spu_buff_add(ESCAPE_CODE, spu);
538                 spu_buff_add(SPU_PROFILING_CODE, spu);
539                 spu_buff_add(num_spu_nodes, spu);
540         }
541         spin_unlock_irqrestore(&buffer_lock, flags);
542
543         for (spu = 0; spu < num_spu_nodes; spu++) {
544                 spu_buff[spu].ctx_sw_seen = 0;
545                 spu_buff[spu].last_guard_val = 0;
546         }
547
548         /* Register for SPU events  */
549         register_ret = spu_switch_event_register(&spu_active);
550         if (register_ret) {
551                 ret = SYNC_START_ERROR;
552                 goto out;
553         }
554
555         pr_debug("spu_sync_start -- running.\n");
556 out:
557         return ret;
558 }
559
560 /* Record SPU program counter samples to the oprofile event buffer. */
561 void spu_sync_buffer(int spu_num, unsigned int *samples,
562                      int num_samples)
563 {
564         unsigned long long file_offset;
565         unsigned long flags;
566         int i;
567         struct vma_to_fileoffset_map *map;
568         struct spu *the_spu;
569         unsigned long long spu_num_ll = spu_num;
570         unsigned long long spu_num_shifted = spu_num_ll << 32;
571         struct cached_info *c_info;
572
573         /* We need to obtain the cache_lock here because it's
574          * possible that after getting the cached_info, the SPU job
575          * corresponding to this cached_info may end, thus resulting
576          * in the destruction of the cached_info.
577          */
578         spin_lock_irqsave(&cache_lock, flags);
579         c_info = get_cached_info(NULL, spu_num);
580         if (!c_info) {
581                 /* This legitimately happens when the SPU task ends before all
582                  * samples are recorded.
583                  * No big deal -- so we just drop a few samples.
584                  */
585                 pr_debug("SPU_PROF: No cached SPU contex "
586                           "for SPU #%d. Dropping samples.\n", spu_num);
587                 goto out;
588         }
589
590         map = c_info->map;
591         the_spu = c_info->the_spu;
592         spin_lock(&buffer_lock);
593         for (i = 0; i < num_samples; i++) {
594                 unsigned int sample = *(samples+i);
595                 int grd_val = 0;
596                 file_offset = 0;
597                 if (sample == 0)
598                         continue;
599                 file_offset = vma_map_lookup( map, sample, the_spu, &grd_val);
600
601                 /* If overlays are used by this SPU application, the guard
602                  * value is non-zero, indicating which overlay section is in
603                  * use.  We need to discard samples taken during the time
604                  * period which an overlay occurs (i.e., guard value changes).
605                  */
606                 if (grd_val && grd_val != spu_buff[spu_num].last_guard_val) {
607                         spu_buff[spu_num].last_guard_val = grd_val;
608                         /* Drop the rest of the samples. */
609                         break;
610                 }
611
612                 /* We must ensure that the SPU context switch has been written
613                  * out before samples for the SPU.  Otherwise, the SPU context
614                  * information is not available and the postprocessing of the
615                  * SPU PC will fail with no available anonymous map information.
616                  */
617                 if (spu_buff[spu_num].ctx_sw_seen)
618                         spu_buff_add((file_offset | spu_num_shifted),
619                                          spu_num);
620         }
621         spin_unlock(&buffer_lock);
622 out:
623         spin_unlock_irqrestore(&cache_lock, flags);
624 }
625
626
627 int spu_sync_stop(void)
628 {
629         unsigned long flags = 0;
630         int ret;
631         int k;
632
633         ret = spu_switch_event_unregister(&spu_active);
634
635         if (ret)
636                 printk(KERN_ERR "SPU_PROF: "
637                        "%s, line %d: spu_switch_event_unregister "      \
638                        "returned %d\n",
639                        __func__, __LINE__, ret);
640
641         /* flush any remaining data in the per SPU buffers */
642         sync_spu_buff();
643
644         spin_lock_irqsave(&cache_lock, flags);
645         ret = release_cached_info(RELEASE_ALL);
646         spin_unlock_irqrestore(&cache_lock, flags);
647
648         /* remove scheduled work queue item rather then waiting
649          * for every queued entry to execute.  Then flush pending
650          * system wide buffer to event buffer.
651          */
652         cancel_delayed_work(&spu_work);
653
654         for (k = 0; k < num_spu_nodes; k++) {
655                 spu_buff[k].ctx_sw_seen = 0;
656
657                 /*
658                  * spu_sys_buff will be null if there was a problem
659                  * allocating the buffer.  Only delete if it exists.
660                  */
661                 kfree(spu_buff[k].buff);
662                 spu_buff[k].buff = 0;
663         }
664         pr_debug("spu_sync_stop -- done.\n");
665         return ret;
666 }
667