perf sched: Fix 'perf sched latency' output on 32-bit systems
[linux-3.10.git] / tools / perf / builtin-sched.c
1 #include "builtin.h"
2 #include "perf.h"
3
4 #include "util/util.h"
5 #include "util/cache.h"
6 #include "util/symbol.h"
7 #include "util/thread.h"
8 #include "util/header.h"
9
10 #include "util/parse-options.h"
11 #include "util/trace-event.h"
12
13 #include "util/debug.h"
14
15 #include <sys/types.h>
16 #include <sys/prctl.h>
17
18 #include <semaphore.h>
19 #include <pthread.h>
20 #include <math.h>
21
22 static char                     const *input_name = "perf.data";
23 static int                      input;
24 static unsigned long            page_size;
25 static unsigned long            mmap_window = 32;
26
27 static unsigned long            total_comm = 0;
28
29 static struct rb_root           threads;
30 static struct thread            *last_match;
31
32 static struct perf_header       *header;
33 static u64                      sample_type;
34
35 static char                     default_sort_order[] = "avg, max, switch, runtime";
36 static char                     *sort_order = default_sort_order;
37
38 #define PR_SET_NAME             15               /* Set process name */
39 #define MAX_CPUS                4096
40
41 #define BUG_ON(x)               assert(!(x))
42
43 static u64                      run_measurement_overhead;
44 static u64                      sleep_measurement_overhead;
45
46 #define COMM_LEN                20
47 #define SYM_LEN                 129
48
49 #define MAX_PID                 65536
50
51 static unsigned long            nr_tasks;
52
53 struct sched_event;
54
55 struct task_desc {
56         unsigned long           nr;
57         unsigned long           pid;
58         char                    comm[COMM_LEN];
59
60         unsigned long           nr_events;
61         unsigned long           curr_event;
62         struct sched_event      **events;
63
64         pthread_t               thread;
65         sem_t                   sleep_sem;
66
67         sem_t                   ready_for_work;
68         sem_t                   work_done_sem;
69
70         u64                     cpu_usage;
71 };
72
73 enum sched_event_type {
74         SCHED_EVENT_RUN,
75         SCHED_EVENT_SLEEP,
76         SCHED_EVENT_WAKEUP,
77 };
78
79 struct sched_event {
80         enum sched_event_type   type;
81         u64                     timestamp;
82         u64                     duration;
83         unsigned long           nr;
84         int                     specific_wait;
85         sem_t                   *wait_sem;
86         struct task_desc        *wakee;
87 };
88
89 static struct task_desc         *pid_to_task[MAX_PID];
90
91 static struct task_desc         **tasks;
92
93 static pthread_mutex_t          start_work_mutex = PTHREAD_MUTEX_INITIALIZER;
94 static u64                      start_time;
95
96 static pthread_mutex_t          work_done_wait_mutex = PTHREAD_MUTEX_INITIALIZER;
97
98 static unsigned long            nr_run_events;
99 static unsigned long            nr_sleep_events;
100 static unsigned long            nr_wakeup_events;
101
102 static unsigned long            nr_sleep_corrections;
103 static unsigned long            nr_run_events_optimized;
104
105 static unsigned long            targetless_wakeups;
106 static unsigned long            multitarget_wakeups;
107
108 static u64                      cpu_usage;
109 static u64                      runavg_cpu_usage;
110 static u64                      parent_cpu_usage;
111 static u64                      runavg_parent_cpu_usage;
112
113 static unsigned long            nr_runs;
114 static u64                      sum_runtime;
115 static u64                      sum_fluct;
116 static u64                      run_avg;
117
118 static unsigned long            replay_repeat = 10;
119 static unsigned long            nr_timestamps;
120 static unsigned long            unordered_timestamps;
121
122 #define TASK_STATE_TO_CHAR_STR "RSDTtZX"
123
124 enum thread_state {
125         THREAD_SLEEPING = 0,
126         THREAD_WAIT_CPU,
127         THREAD_SCHED_IN,
128         THREAD_IGNORE
129 };
130
131 struct work_atom {
132         struct list_head        list;
133         enum thread_state       state;
134         u64                     sched_out_time;
135         u64                     wake_up_time;
136         u64                     sched_in_time;
137         u64                     runtime;
138 };
139
140 struct task_atoms {
141         struct list_head        atom_list;
142         struct thread           *thread;
143         struct rb_node          node;
144         u64                     max_lat;
145         u64                     total_lat;
146         u64                     nb_atoms;
147         u64                     total_runtime;
148 };
149
150 typedef int (*sort_fn_t)(struct task_atoms *, struct task_atoms *);
151
152 static struct rb_root           atom_root, sorted_atom_root;
153
154 static u64                      all_runtime;
155 static u64                      all_count;
156
157 static int read_events(void);
158
159
160 static u64 get_nsecs(void)
161 {
162         struct timespec ts;
163
164         clock_gettime(CLOCK_MONOTONIC, &ts);
165
166         return ts.tv_sec * 1000000000ULL + ts.tv_nsec;
167 }
168
169 static void burn_nsecs(u64 nsecs)
170 {
171         u64 T0 = get_nsecs(), T1;
172
173         do {
174                 T1 = get_nsecs();
175         } while (T1 + run_measurement_overhead < T0 + nsecs);
176 }
177
178 static void sleep_nsecs(u64 nsecs)
179 {
180         struct timespec ts;
181
182         ts.tv_nsec = nsecs % 999999999;
183         ts.tv_sec = nsecs / 999999999;
184
185         nanosleep(&ts, NULL);
186 }
187
188 static void calibrate_run_measurement_overhead(void)
189 {
190         u64 T0, T1, delta, min_delta = 1000000000ULL;
191         int i;
192
193         for (i = 0; i < 10; i++) {
194                 T0 = get_nsecs();
195                 burn_nsecs(0);
196                 T1 = get_nsecs();
197                 delta = T1-T0;
198                 min_delta = min(min_delta, delta);
199         }
200         run_measurement_overhead = min_delta;
201
202         printf("run measurement overhead: %Ld nsecs\n", min_delta);
203 }
204
205 static void calibrate_sleep_measurement_overhead(void)
206 {
207         u64 T0, T1, delta, min_delta = 1000000000ULL;
208         int i;
209
210         for (i = 0; i < 10; i++) {
211                 T0 = get_nsecs();
212                 sleep_nsecs(10000);
213                 T1 = get_nsecs();
214                 delta = T1-T0;
215                 min_delta = min(min_delta, delta);
216         }
217         min_delta -= 10000;
218         sleep_measurement_overhead = min_delta;
219
220         printf("sleep measurement overhead: %Ld nsecs\n", min_delta);
221 }
222
223 static struct sched_event *
224 get_new_event(struct task_desc *task, u64 timestamp)
225 {
226         struct sched_event *event = calloc(1, sizeof(*event));
227         unsigned long idx = task->nr_events;
228         size_t size;
229
230         event->timestamp = timestamp;
231         event->nr = idx;
232
233         task->nr_events++;
234         size = sizeof(struct sched_event *) * task->nr_events;
235         task->events = realloc(task->events, size);
236         BUG_ON(!task->events);
237
238         task->events[idx] = event;
239
240         return event;
241 }
242
243 static struct sched_event *last_event(struct task_desc *task)
244 {
245         if (!task->nr_events)
246                 return NULL;
247
248         return task->events[task->nr_events - 1];
249 }
250
251 static void
252 add_sched_event_run(struct task_desc *task, u64 timestamp, u64 duration)
253 {
254         struct sched_event *event, *curr_event = last_event(task);
255
256         /*
257          * optimize an existing RUN event by merging this one
258          * to it:
259          */
260         if (curr_event && curr_event->type == SCHED_EVENT_RUN) {
261                 nr_run_events_optimized++;
262                 curr_event->duration += duration;
263                 return;
264         }
265
266         event = get_new_event(task, timestamp);
267
268         event->type = SCHED_EVENT_RUN;
269         event->duration = duration;
270
271         nr_run_events++;
272 }
273
274 static void
275 add_sched_event_wakeup(struct task_desc *task, u64 timestamp,
276                        struct task_desc *wakee)
277 {
278         struct sched_event *event, *wakee_event;
279
280         event = get_new_event(task, timestamp);
281         event->type = SCHED_EVENT_WAKEUP;
282         event->wakee = wakee;
283
284         wakee_event = last_event(wakee);
285         if (!wakee_event || wakee_event->type != SCHED_EVENT_SLEEP) {
286                 targetless_wakeups++;
287                 return;
288         }
289         if (wakee_event->wait_sem) {
290                 multitarget_wakeups++;
291                 return;
292         }
293
294         wakee_event->wait_sem = calloc(1, sizeof(*wakee_event->wait_sem));
295         sem_init(wakee_event->wait_sem, 0, 0);
296         wakee_event->specific_wait = 1;
297         event->wait_sem = wakee_event->wait_sem;
298
299         nr_wakeup_events++;
300 }
301
302 static void
303 add_sched_event_sleep(struct task_desc *task, u64 timestamp,
304                       u64 task_state __used)
305 {
306         struct sched_event *event = get_new_event(task, timestamp);
307
308         event->type = SCHED_EVENT_SLEEP;
309
310         nr_sleep_events++;
311 }
312
313 static struct task_desc *register_pid(unsigned long pid, const char *comm)
314 {
315         struct task_desc *task;
316
317         BUG_ON(pid >= MAX_PID);
318
319         task = pid_to_task[pid];
320
321         if (task)
322                 return task;
323
324         task = calloc(1, sizeof(*task));
325         task->pid = pid;
326         task->nr = nr_tasks;
327         strcpy(task->comm, comm);
328         /*
329          * every task starts in sleeping state - this gets ignored
330          * if there's no wakeup pointing to this sleep state:
331          */
332         add_sched_event_sleep(task, 0, 0);
333
334         pid_to_task[pid] = task;
335         nr_tasks++;
336         tasks = realloc(tasks, nr_tasks*sizeof(struct task_task *));
337         BUG_ON(!tasks);
338         tasks[task->nr] = task;
339
340         if (verbose)
341                 printf("registered task #%ld, PID %ld (%s)\n", nr_tasks, pid, comm);
342
343         return task;
344 }
345
346
347 static void print_task_traces(void)
348 {
349         struct task_desc *task;
350         unsigned long i;
351
352         for (i = 0; i < nr_tasks; i++) {
353                 task = tasks[i];
354                 printf("task %6ld (%20s:%10ld), nr_events: %ld\n",
355                         task->nr, task->comm, task->pid, task->nr_events);
356         }
357 }
358
359 static void add_cross_task_wakeups(void)
360 {
361         struct task_desc *task1, *task2;
362         unsigned long i, j;
363
364         for (i = 0; i < nr_tasks; i++) {
365                 task1 = tasks[i];
366                 j = i + 1;
367                 if (j == nr_tasks)
368                         j = 0;
369                 task2 = tasks[j];
370                 add_sched_event_wakeup(task1, 0, task2);
371         }
372 }
373
374 static void
375 process_sched_event(struct task_desc *this_task __used, struct sched_event *event)
376 {
377         int ret = 0;
378         u64 now;
379         long long delta;
380
381         now = get_nsecs();
382         delta = start_time + event->timestamp - now;
383
384         switch (event->type) {
385                 case SCHED_EVENT_RUN:
386                         burn_nsecs(event->duration);
387                         break;
388                 case SCHED_EVENT_SLEEP:
389                         if (event->wait_sem)
390                                 ret = sem_wait(event->wait_sem);
391                         BUG_ON(ret);
392                         break;
393                 case SCHED_EVENT_WAKEUP:
394                         if (event->wait_sem)
395                                 ret = sem_post(event->wait_sem);
396                         BUG_ON(ret);
397                         break;
398                 default:
399                         BUG_ON(1);
400         }
401 }
402
403 static u64 get_cpu_usage_nsec_parent(void)
404 {
405         struct rusage ru;
406         u64 sum;
407         int err;
408
409         err = getrusage(RUSAGE_SELF, &ru);
410         BUG_ON(err);
411
412         sum =  ru.ru_utime.tv_sec*1e9 + ru.ru_utime.tv_usec*1e3;
413         sum += ru.ru_stime.tv_sec*1e9 + ru.ru_stime.tv_usec*1e3;
414
415         return sum;
416 }
417
418 static u64 get_cpu_usage_nsec_self(void)
419 {
420         char filename [] = "/proc/1234567890/sched";
421         unsigned long msecs, nsecs;
422         char *line = NULL;
423         u64 total = 0;
424         size_t len = 0;
425         ssize_t chars;
426         FILE *file;
427         int ret;
428
429         sprintf(filename, "/proc/%d/sched", getpid());
430         file = fopen(filename, "r");
431         BUG_ON(!file);
432
433         while ((chars = getline(&line, &len, file)) != -1) {
434                 ret = sscanf(line, "se.sum_exec_runtime : %ld.%06ld\n",
435                         &msecs, &nsecs);
436                 if (ret == 2) {
437                         total = msecs*1e6 + nsecs;
438                         break;
439                 }
440         }
441         if (line)
442                 free(line);
443         fclose(file);
444
445         return total;
446 }
447
448 static void *thread_func(void *ctx)
449 {
450         struct task_desc *this_task = ctx;
451         u64 cpu_usage_0, cpu_usage_1;
452         unsigned long i, ret;
453         char comm2[22];
454
455         sprintf(comm2, ":%s", this_task->comm);
456         prctl(PR_SET_NAME, comm2);
457
458 again:
459         ret = sem_post(&this_task->ready_for_work);
460         BUG_ON(ret);
461         ret = pthread_mutex_lock(&start_work_mutex);
462         BUG_ON(ret);
463         ret = pthread_mutex_unlock(&start_work_mutex);
464         BUG_ON(ret);
465
466         cpu_usage_0 = get_cpu_usage_nsec_self();
467
468         for (i = 0; i < this_task->nr_events; i++) {
469                 this_task->curr_event = i;
470                 process_sched_event(this_task, this_task->events[i]);
471         }
472
473         cpu_usage_1 = get_cpu_usage_nsec_self();
474         this_task->cpu_usage = cpu_usage_1 - cpu_usage_0;
475
476         ret = sem_post(&this_task->work_done_sem);
477         BUG_ON(ret);
478
479         ret = pthread_mutex_lock(&work_done_wait_mutex);
480         BUG_ON(ret);
481         ret = pthread_mutex_unlock(&work_done_wait_mutex);
482         BUG_ON(ret);
483
484         goto again;
485 }
486
487 static void create_tasks(void)
488 {
489         struct task_desc *task;
490         pthread_attr_t attr;
491         unsigned long i;
492         int err;
493
494         err = pthread_attr_init(&attr);
495         BUG_ON(err);
496         err = pthread_attr_setstacksize(&attr, (size_t)(16*1024));
497         BUG_ON(err);
498         err = pthread_mutex_lock(&start_work_mutex);
499         BUG_ON(err);
500         err = pthread_mutex_lock(&work_done_wait_mutex);
501         BUG_ON(err);
502         for (i = 0; i < nr_tasks; i++) {
503                 task = tasks[i];
504                 sem_init(&task->sleep_sem, 0, 0);
505                 sem_init(&task->ready_for_work, 0, 0);
506                 sem_init(&task->work_done_sem, 0, 0);
507                 task->curr_event = 0;
508                 err = pthread_create(&task->thread, &attr, thread_func, task);
509                 BUG_ON(err);
510         }
511 }
512
513 static void wait_for_tasks(void)
514 {
515         u64 cpu_usage_0, cpu_usage_1;
516         struct task_desc *task;
517         unsigned long i, ret;
518
519         start_time = get_nsecs();
520         cpu_usage = 0;
521         pthread_mutex_unlock(&work_done_wait_mutex);
522
523         for (i = 0; i < nr_tasks; i++) {
524                 task = tasks[i];
525                 ret = sem_wait(&task->ready_for_work);
526                 BUG_ON(ret);
527                 sem_init(&task->ready_for_work, 0, 0);
528         }
529         ret = pthread_mutex_lock(&work_done_wait_mutex);
530         BUG_ON(ret);
531
532         cpu_usage_0 = get_cpu_usage_nsec_parent();
533
534         pthread_mutex_unlock(&start_work_mutex);
535
536         for (i = 0; i < nr_tasks; i++) {
537                 task = tasks[i];
538                 ret = sem_wait(&task->work_done_sem);
539                 BUG_ON(ret);
540                 sem_init(&task->work_done_sem, 0, 0);
541                 cpu_usage += task->cpu_usage;
542                 task->cpu_usage = 0;
543         }
544
545         cpu_usage_1 = get_cpu_usage_nsec_parent();
546         if (!runavg_cpu_usage)
547                 runavg_cpu_usage = cpu_usage;
548         runavg_cpu_usage = (runavg_cpu_usage*9 + cpu_usage)/10;
549
550         parent_cpu_usage = cpu_usage_1 - cpu_usage_0;
551         if (!runavg_parent_cpu_usage)
552                 runavg_parent_cpu_usage = parent_cpu_usage;
553         runavg_parent_cpu_usage = (runavg_parent_cpu_usage*9 +
554                                    parent_cpu_usage)/10;
555
556         ret = pthread_mutex_lock(&start_work_mutex);
557         BUG_ON(ret);
558
559         for (i = 0; i < nr_tasks; i++) {
560                 task = tasks[i];
561                 sem_init(&task->sleep_sem, 0, 0);
562                 task->curr_event = 0;
563         }
564 }
565
566 static void run_one_test(void)
567 {
568         u64 T0, T1, delta, avg_delta, fluct, std_dev;
569
570         T0 = get_nsecs();
571         wait_for_tasks();
572         T1 = get_nsecs();
573
574         delta = T1 - T0;
575         sum_runtime += delta;
576         nr_runs++;
577
578         avg_delta = sum_runtime / nr_runs;
579         if (delta < avg_delta)
580                 fluct = avg_delta - delta;
581         else
582                 fluct = delta - avg_delta;
583         sum_fluct += fluct;
584         std_dev = sum_fluct / nr_runs / sqrt(nr_runs);
585         if (!run_avg)
586                 run_avg = delta;
587         run_avg = (run_avg*9 + delta)/10;
588
589         printf("#%-3ld: %0.3f, ",
590                 nr_runs, (double)delta/1000000.0);
591
592         printf("ravg: %0.2f, ",
593                 (double)run_avg/1e6);
594
595         printf("cpu: %0.2f / %0.2f",
596                 (double)cpu_usage/1e6, (double)runavg_cpu_usage/1e6);
597
598 #if 0
599         /*
600          * rusage statistics done by the parent, these are less
601          * accurate than the sum_exec_runtime based statistics:
602          */
603         printf(" [%0.2f / %0.2f]",
604                 (double)parent_cpu_usage/1e6,
605                 (double)runavg_parent_cpu_usage/1e6);
606 #endif
607
608         printf("\n");
609
610         if (nr_sleep_corrections)
611                 printf(" (%ld sleep corrections)\n", nr_sleep_corrections);
612         nr_sleep_corrections = 0;
613 }
614
615 static void test_calibrations(void)
616 {
617         u64 T0, T1;
618
619         T0 = get_nsecs();
620         burn_nsecs(1e6);
621         T1 = get_nsecs();
622
623         printf("the run test took %Ld nsecs\n", T1-T0);
624
625         T0 = get_nsecs();
626         sleep_nsecs(1e6);
627         T1 = get_nsecs();
628
629         printf("the sleep test took %Ld nsecs\n", T1-T0);
630 }
631
632 static void __cmd_replay(void)
633 {
634         unsigned long i;
635
636         calibrate_run_measurement_overhead();
637         calibrate_sleep_measurement_overhead();
638
639         test_calibrations();
640
641         read_events();
642
643         printf("nr_run_events:        %ld\n", nr_run_events);
644         printf("nr_sleep_events:      %ld\n", nr_sleep_events);
645         printf("nr_wakeup_events:     %ld\n", nr_wakeup_events);
646
647         if (targetless_wakeups)
648                 printf("target-less wakeups:  %ld\n", targetless_wakeups);
649         if (multitarget_wakeups)
650                 printf("multi-target wakeups: %ld\n", multitarget_wakeups);
651         if (nr_run_events_optimized)
652                 printf("run events optimized: %ld\n",
653                         nr_run_events_optimized);
654
655         print_task_traces();
656         add_cross_task_wakeups();
657
658         create_tasks();
659         printf("------------------------------------------------------------\n");
660         for (i = 0; i < replay_repeat; i++)
661                 run_one_test();
662 }
663
664 static int
665 process_comm_event(event_t *event, unsigned long offset, unsigned long head)
666 {
667         struct thread *thread;
668
669         thread = threads__findnew(event->comm.pid, &threads, &last_match);
670
671         dump_printf("%p [%p]: PERF_EVENT_COMM: %s:%d\n",
672                 (void *)(offset + head),
673                 (void *)(long)(event->header.size),
674                 event->comm.comm, event->comm.pid);
675
676         if (thread == NULL ||
677             thread__set_comm(thread, event->comm.comm)) {
678                 dump_printf("problem processing PERF_EVENT_COMM, skipping event.\n");
679                 return -1;
680         }
681         total_comm++;
682
683         return 0;
684 }
685
686
687 struct raw_event_sample {
688         u32 size;
689         char data[0];
690 };
691
692 #define FILL_FIELD(ptr, field, event, data)     \
693         ptr.field = (typeof(ptr.field)) raw_field_value(event, #field, data)
694
695 #define FILL_ARRAY(ptr, array, event, data)                     \
696 do {                                                            \
697         void *__array = raw_field_ptr(event, #array, data);     \
698         memcpy(ptr.array, __array, sizeof(ptr.array));  \
699 } while(0)
700
701 #define FILL_COMMON_FIELDS(ptr, event, data)                    \
702 do {                                                            \
703         FILL_FIELD(ptr, common_type, event, data);              \
704         FILL_FIELD(ptr, common_flags, event, data);             \
705         FILL_FIELD(ptr, common_preempt_count, event, data);     \
706         FILL_FIELD(ptr, common_pid, event, data);               \
707         FILL_FIELD(ptr, common_tgid, event, data);              \
708 } while (0)
709
710
711
712 struct trace_switch_event {
713         u32 size;
714
715         u16 common_type;
716         u8 common_flags;
717         u8 common_preempt_count;
718         u32 common_pid;
719         u32 common_tgid;
720
721         char prev_comm[16];
722         u32 prev_pid;
723         u32 prev_prio;
724         u64 prev_state;
725         char next_comm[16];
726         u32 next_pid;
727         u32 next_prio;
728 };
729
730
731 struct trace_wakeup_event {
732         u32 size;
733
734         u16 common_type;
735         u8 common_flags;
736         u8 common_preempt_count;
737         u32 common_pid;
738         u32 common_tgid;
739
740         char comm[16];
741         u32 pid;
742
743         u32 prio;
744         u32 success;
745         u32 cpu;
746 };
747
748 struct trace_fork_event {
749         u32 size;
750
751         u16 common_type;
752         u8 common_flags;
753         u8 common_preempt_count;
754         u32 common_pid;
755         u32 common_tgid;
756
757         char parent_comm[16];
758         u32 parent_pid;
759         char child_comm[16];
760         u32 child_pid;
761 };
762
763 struct trace_sched_handler {
764         void (*switch_event)(struct trace_switch_event *,
765                              struct event *,
766                              int cpu,
767                              u64 timestamp,
768                              struct thread *thread);
769
770         void (*wakeup_event)(struct trace_wakeup_event *,
771                              struct event *,
772                              int cpu,
773                              u64 timestamp,
774                              struct thread *thread);
775
776         void (*fork_event)(struct trace_fork_event *,
777                            struct event *,
778                            int cpu,
779                            u64 timestamp,
780                            struct thread *thread);
781 };
782
783
784 static void
785 replay_wakeup_event(struct trace_wakeup_event *wakeup_event,
786                     struct event *event,
787                     int cpu __used,
788                     u64 timestamp __used,
789                     struct thread *thread __used)
790 {
791         struct task_desc *waker, *wakee;
792
793         if (verbose) {
794                 printf("sched_wakeup event %p\n", event);
795
796                 printf(" ... pid %d woke up %s/%d\n",
797                         wakeup_event->common_pid,
798                         wakeup_event->comm,
799                         wakeup_event->pid);
800         }
801
802         waker = register_pid(wakeup_event->common_pid, "<unknown>");
803         wakee = register_pid(wakeup_event->pid, wakeup_event->comm);
804
805         add_sched_event_wakeup(waker, timestamp, wakee);
806 }
807
808 static u64 cpu_last_switched[MAX_CPUS];
809
810 static void
811 replay_switch_event(struct trace_switch_event *switch_event,
812                     struct event *event,
813                     int cpu,
814                     u64 timestamp,
815                     struct thread *thread __used)
816 {
817         struct task_desc *prev, *next;
818         u64 timestamp0;
819         s64 delta;
820
821         if (verbose)
822                 printf("sched_switch event %p\n", event);
823
824         if (cpu >= MAX_CPUS || cpu < 0)
825                 return;
826
827         timestamp0 = cpu_last_switched[cpu];
828         if (timestamp0)
829                 delta = timestamp - timestamp0;
830         else
831                 delta = 0;
832
833         if (delta < 0)
834                 die("hm, delta: %Ld < 0 ?\n", delta);
835
836         if (verbose) {
837                 printf(" ... switch from %s/%d to %s/%d [ran %Ld nsecs]\n",
838                         switch_event->prev_comm, switch_event->prev_pid,
839                         switch_event->next_comm, switch_event->next_pid,
840                         delta);
841         }
842
843         prev = register_pid(switch_event->prev_pid, switch_event->prev_comm);
844         next = register_pid(switch_event->next_pid, switch_event->next_comm);
845
846         cpu_last_switched[cpu] = timestamp;
847
848         add_sched_event_run(prev, timestamp, delta);
849         add_sched_event_sleep(prev, timestamp, switch_event->prev_state);
850 }
851
852
853 static void
854 replay_fork_event(struct trace_fork_event *fork_event,
855                   struct event *event,
856                   int cpu __used,
857                   u64 timestamp __used,
858                   struct thread *thread __used)
859 {
860         if (verbose) {
861                 printf("sched_fork event %p\n", event);
862                 printf("... parent: %s/%d\n", fork_event->parent_comm, fork_event->parent_pid);
863                 printf("...  child: %s/%d\n", fork_event->child_comm, fork_event->child_pid);
864         }
865         register_pid(fork_event->parent_pid, fork_event->parent_comm);
866         register_pid(fork_event->child_pid, fork_event->child_comm);
867 }
868
869 static struct trace_sched_handler replay_ops  = {
870         .wakeup_event           = replay_wakeup_event,
871         .switch_event           = replay_switch_event,
872         .fork_event             = replay_fork_event,
873 };
874
875 struct sort_dimension {
876         const char              *name;
877         sort_fn_t               cmp;
878         struct list_head        list;
879 };
880
881 static LIST_HEAD(cmp_pid);
882
883 static int
884 thread_lat_cmp(struct list_head *list, struct task_atoms *l, struct task_atoms *r)
885 {
886         struct sort_dimension *sort;
887         int ret = 0;
888
889         BUG_ON(list_empty(list));
890
891         list_for_each_entry(sort, list, list) {
892                 ret = sort->cmp(l, r);
893                 if (ret)
894                         return ret;
895         }
896
897         return ret;
898 }
899
900 static struct task_atoms *
901 thread_atoms_search(struct rb_root *root, struct thread *thread,
902                          struct list_head *sort_list)
903 {
904         struct rb_node *node = root->rb_node;
905         struct task_atoms key = { .thread = thread };
906
907         while (node) {
908                 struct task_atoms *atoms;
909                 int cmp;
910
911                 atoms = container_of(node, struct task_atoms, node);
912
913                 cmp = thread_lat_cmp(sort_list, &key, atoms);
914                 if (cmp > 0)
915                         node = node->rb_left;
916                 else if (cmp < 0)
917                         node = node->rb_right;
918                 else {
919                         BUG_ON(thread != atoms->thread);
920                         return atoms;
921                 }
922         }
923         return NULL;
924 }
925
926 static void
927 __thread_latency_insert(struct rb_root *root, struct task_atoms *data,
928                          struct list_head *sort_list)
929 {
930         struct rb_node **new = &(root->rb_node), *parent = NULL;
931
932         while (*new) {
933                 struct task_atoms *this;
934                 int cmp;
935
936                 this = container_of(*new, struct task_atoms, node);
937                 parent = *new;
938
939                 cmp = thread_lat_cmp(sort_list, data, this);
940
941                 if (cmp > 0)
942                         new = &((*new)->rb_left);
943                 else
944                         new = &((*new)->rb_right);
945         }
946
947         rb_link_node(&data->node, parent, new);
948         rb_insert_color(&data->node, root);
949 }
950
951 static void thread_atoms_insert(struct thread *thread)
952 {
953         struct task_atoms *atoms;
954
955         atoms = calloc(sizeof(*atoms), 1);
956         if (!atoms)
957                 die("No memory");
958
959         atoms->thread = thread;
960         INIT_LIST_HEAD(&atoms->atom_list);
961         __thread_latency_insert(&atom_root, atoms, &cmp_pid);
962 }
963
964 static void
965 latency_fork_event(struct trace_fork_event *fork_event __used,
966                    struct event *event __used,
967                    int cpu __used,
968                    u64 timestamp __used,
969                    struct thread *thread __used)
970 {
971         /* should insert the newcomer */
972 }
973
974 __used
975 static char sched_out_state(struct trace_switch_event *switch_event)
976 {
977         const char *str = TASK_STATE_TO_CHAR_STR;
978
979         return str[switch_event->prev_state];
980 }
981
982 static void
983 lat_sched_out(struct task_atoms *atoms,
984               struct trace_switch_event *switch_event __used,
985               u64 delta,
986               u64 timestamp)
987 {
988         struct work_atom *atom;
989
990         atom = calloc(sizeof(*atom), 1);
991         if (!atom)
992                 die("Non memory");
993
994         atom->sched_out_time = timestamp;
995
996         if (sched_out_state(switch_event) == 'R') {
997                 atom->state = THREAD_WAIT_CPU;
998                 atom->wake_up_time = atom->sched_out_time;
999         }
1000
1001         atom->runtime = delta;
1002         list_add_tail(&atom->list, &atoms->atom_list);
1003 }
1004
1005 static void
1006 lat_sched_in(struct task_atoms *atoms, u64 timestamp)
1007 {
1008         struct work_atom *atom;
1009         u64 delta;
1010
1011         if (list_empty(&atoms->atom_list))
1012                 return;
1013
1014         atom = list_entry(atoms->atom_list.prev, struct work_atom, list);
1015
1016         if (atom->state != THREAD_WAIT_CPU)
1017                 return;
1018
1019         if (timestamp < atom->wake_up_time) {
1020                 atom->state = THREAD_IGNORE;
1021                 return;
1022         }
1023
1024         atom->state = THREAD_SCHED_IN;
1025         atom->sched_in_time = timestamp;
1026
1027         delta = atom->sched_in_time - atom->wake_up_time;
1028         atoms->total_lat += delta;
1029         if (delta > atoms->max_lat)
1030                 atoms->max_lat = delta;
1031         atoms->nb_atoms++;
1032         atoms->total_runtime += atom->runtime;
1033 }
1034
1035 static void
1036 latency_switch_event(struct trace_switch_event *switch_event,
1037                      struct event *event __used,
1038                      int cpu,
1039                      u64 timestamp,
1040                      struct thread *thread __used)
1041 {
1042         struct task_atoms *out_atoms, *in_atoms;
1043         struct thread *sched_out, *sched_in;
1044         u64 timestamp0;
1045         s64 delta;
1046
1047         if (cpu >= MAX_CPUS || cpu < 0)
1048                 return;
1049
1050         timestamp0 = cpu_last_switched[cpu];
1051         cpu_last_switched[cpu] = timestamp;
1052         if (timestamp0)
1053                 delta = timestamp - timestamp0;
1054         else
1055                 delta = 0;
1056
1057         if (delta < 0)
1058                 die("hm, delta: %Ld < 0 ?\n", delta);
1059
1060
1061         sched_out = threads__findnew(switch_event->prev_pid, &threads, &last_match);
1062         sched_in = threads__findnew(switch_event->next_pid, &threads, &last_match);
1063
1064         in_atoms = thread_atoms_search(&atom_root, sched_in, &cmp_pid);
1065         if (!in_atoms) {
1066                 thread_atoms_insert(sched_in);
1067                 in_atoms = thread_atoms_search(&atom_root, sched_in, &cmp_pid);
1068                 if (!in_atoms)
1069                         die("in-atom: Internal tree error");
1070         }
1071
1072         out_atoms = thread_atoms_search(&atom_root, sched_out, &cmp_pid);
1073         if (!out_atoms) {
1074                 thread_atoms_insert(sched_out);
1075                 out_atoms = thread_atoms_search(&atom_root, sched_out, &cmp_pid);
1076                 if (!out_atoms)
1077                         die("out-atom: Internal tree error");
1078         }
1079
1080         lat_sched_in(in_atoms, timestamp);
1081         lat_sched_out(out_atoms, switch_event, delta, timestamp);
1082 }
1083
1084 static void
1085 latency_wakeup_event(struct trace_wakeup_event *wakeup_event,
1086                      struct event *event __used,
1087                      int cpu __used,
1088                      u64 timestamp,
1089                      struct thread *thread __used)
1090 {
1091         struct task_atoms *atoms;
1092         struct work_atom *atom;
1093         struct thread *wakee;
1094
1095         /* Note for later, it may be interesting to observe the failing cases */
1096         if (!wakeup_event->success)
1097                 return;
1098
1099         wakee = threads__findnew(wakeup_event->pid, &threads, &last_match);
1100         atoms = thread_atoms_search(&atom_root, wakee, &cmp_pid);
1101         if (!atoms) {
1102                 thread_atoms_insert(wakee);
1103                 return;
1104         }
1105
1106         if (list_empty(&atoms->atom_list))
1107                 return;
1108
1109         atom = list_entry(atoms->atom_list.prev, struct work_atom, list);
1110
1111         if (atom->state != THREAD_SLEEPING)
1112                 return;
1113
1114         nr_timestamps++;
1115         if (atom->sched_out_time > timestamp) {
1116                 unordered_timestamps++;
1117                 return;
1118         }
1119
1120         atom->state = THREAD_WAIT_CPU;
1121         atom->wake_up_time = timestamp;
1122 }
1123
1124 static struct trace_sched_handler lat_ops  = {
1125         .wakeup_event           = latency_wakeup_event,
1126         .switch_event           = latency_switch_event,
1127         .fork_event             = latency_fork_event,
1128 };
1129
1130 static void output_lat_thread(struct task_atoms *atom_list)
1131 {
1132         int i;
1133         int ret;
1134         u64 avg;
1135
1136         if (!atom_list->nb_atoms)
1137                 return;
1138         /*
1139          * Ignore idle threads:
1140          */
1141         if (!atom_list->thread->pid)
1142                 return;
1143
1144         all_runtime += atom_list->total_runtime;
1145         all_count += atom_list->nb_atoms;
1146
1147         ret = printf(" %s ", atom_list->thread->comm);
1148
1149         for (i = 0; i < 19 - ret; i++)
1150                 printf(" ");
1151
1152         avg = atom_list->total_lat / atom_list->nb_atoms;
1153
1154         printf("|%9.3f ms |%9llu | avg:%9.3f ms | max:%9.3f ms |\n",
1155               (double)atom_list->total_runtime / 1e6,
1156                  atom_list->nb_atoms, (double)avg / 1e6,
1157                  (double)atom_list->max_lat / 1e6);
1158 }
1159
1160 static int pid_cmp(struct task_atoms *l, struct task_atoms *r)
1161 {
1162         if (l->thread->pid < r->thread->pid)
1163                 return -1;
1164         if (l->thread->pid > r->thread->pid)
1165                 return 1;
1166
1167         return 0;
1168 }
1169
1170 static struct sort_dimension pid_sort_dimension = {
1171         .name                   = "pid",
1172         .cmp                    = pid_cmp,
1173 };
1174
1175 static int avg_cmp(struct task_atoms *l, struct task_atoms *r)
1176 {
1177         u64 avgl, avgr;
1178
1179         if (!l->nb_atoms)
1180                 return -1;
1181
1182         if (!r->nb_atoms)
1183                 return 1;
1184
1185         avgl = l->total_lat / l->nb_atoms;
1186         avgr = r->total_lat / r->nb_atoms;
1187
1188         if (avgl < avgr)
1189                 return -1;
1190         if (avgl > avgr)
1191                 return 1;
1192
1193         return 0;
1194 }
1195
1196 static struct sort_dimension avg_sort_dimension = {
1197         .name                   = "avg",
1198         .cmp                    = avg_cmp,
1199 };
1200
1201 static int max_cmp(struct task_atoms *l, struct task_atoms *r)
1202 {
1203         if (l->max_lat < r->max_lat)
1204                 return -1;
1205         if (l->max_lat > r->max_lat)
1206                 return 1;
1207
1208         return 0;
1209 }
1210
1211 static struct sort_dimension max_sort_dimension = {
1212         .name                   = "max",
1213         .cmp                    = max_cmp,
1214 };
1215
1216 static int switch_cmp(struct task_atoms *l, struct task_atoms *r)
1217 {
1218         if (l->nb_atoms < r->nb_atoms)
1219                 return -1;
1220         if (l->nb_atoms > r->nb_atoms)
1221                 return 1;
1222
1223         return 0;
1224 }
1225
1226 static struct sort_dimension switch_sort_dimension = {
1227         .name                   = "switch",
1228         .cmp                    = switch_cmp,
1229 };
1230
1231 static int runtime_cmp(struct task_atoms *l, struct task_atoms *r)
1232 {
1233         if (l->total_runtime < r->total_runtime)
1234                 return -1;
1235         if (l->total_runtime > r->total_runtime)
1236                 return 1;
1237
1238         return 0;
1239 }
1240
1241 static struct sort_dimension runtime_sort_dimension = {
1242         .name                   = "runtime",
1243         .cmp                    = runtime_cmp,
1244 };
1245
1246 static struct sort_dimension *available_sorts[] = {
1247         &pid_sort_dimension,
1248         &avg_sort_dimension,
1249         &max_sort_dimension,
1250         &switch_sort_dimension,
1251         &runtime_sort_dimension,
1252 };
1253
1254 #define NB_AVAILABLE_SORTS      (int)(sizeof(available_sorts) / sizeof(struct sort_dimension *))
1255
1256 static LIST_HEAD(sort_list);
1257
1258 static int sort_dimension__add(char *tok, struct list_head *list)
1259 {
1260         int i;
1261
1262         for (i = 0; i < NB_AVAILABLE_SORTS; i++) {
1263                 if (!strcmp(available_sorts[i]->name, tok)) {
1264                         list_add_tail(&available_sorts[i]->list, list);
1265
1266                         return 0;
1267                 }
1268         }
1269
1270         return -1;
1271 }
1272
1273 static void setup_sorting(void);
1274
1275 static void sort_lat(void)
1276 {
1277         struct rb_node *node;
1278
1279         for (;;) {
1280                 struct task_atoms *data;
1281                 node = rb_first(&atom_root);
1282                 if (!node)
1283                         break;
1284
1285                 rb_erase(node, &atom_root);
1286                 data = rb_entry(node, struct task_atoms, node);
1287                 __thread_latency_insert(&sorted_atom_root, data, &sort_list);
1288         }
1289 }
1290
1291 static void __cmd_lat(void)
1292 {
1293         struct rb_node *next;
1294
1295         setup_pager();
1296         read_events();
1297         sort_lat();
1298
1299         printf("-----------------------------------------------------------------------------------\n");
1300         printf(" Task              |  Runtime ms | Switches | Average delay ms | Maximum delay ms |\n");
1301         printf("-----------------------------------------------------------------------------------\n");
1302
1303         next = rb_first(&sorted_atom_root);
1304
1305         while (next) {
1306                 struct task_atoms *atom_list;
1307
1308                 atom_list = rb_entry(next, struct task_atoms, node);
1309                 output_lat_thread(atom_list);
1310                 next = rb_next(next);
1311         }
1312
1313         printf("-----------------------------------------------------------------------------------\n");
1314         printf(" TOTAL:            |%9.3f ms |%9Ld |",
1315                 (double)all_runtime/1e6, all_count);
1316
1317         if (unordered_timestamps && nr_timestamps) {
1318                 printf(" INFO: %.2f%% unordered events.\n",
1319                         (double)unordered_timestamps/(double)nr_timestamps*100.0);
1320         } else {
1321                 printf("\n");
1322         }
1323
1324         printf("---------------------------------------------\n");
1325 }
1326
1327 static struct trace_sched_handler *trace_handler;
1328
1329 static void
1330 process_sched_wakeup_event(struct raw_event_sample *raw,
1331                            struct event *event,
1332                            int cpu __used,
1333                            u64 timestamp __used,
1334                            struct thread *thread __used)
1335 {
1336         struct trace_wakeup_event wakeup_event;
1337
1338         FILL_COMMON_FIELDS(wakeup_event, event, raw->data);
1339
1340         FILL_ARRAY(wakeup_event, comm, event, raw->data);
1341         FILL_FIELD(wakeup_event, pid, event, raw->data);
1342         FILL_FIELD(wakeup_event, prio, event, raw->data);
1343         FILL_FIELD(wakeup_event, success, event, raw->data);
1344         FILL_FIELD(wakeup_event, cpu, event, raw->data);
1345
1346         trace_handler->wakeup_event(&wakeup_event, event, cpu, timestamp, thread);
1347 }
1348
1349 static void
1350 process_sched_switch_event(struct raw_event_sample *raw,
1351                            struct event *event,
1352                            int cpu __used,
1353                            u64 timestamp __used,
1354                            struct thread *thread __used)
1355 {
1356         struct trace_switch_event switch_event;
1357
1358         FILL_COMMON_FIELDS(switch_event, event, raw->data);
1359
1360         FILL_ARRAY(switch_event, prev_comm, event, raw->data);
1361         FILL_FIELD(switch_event, prev_pid, event, raw->data);
1362         FILL_FIELD(switch_event, prev_prio, event, raw->data);
1363         FILL_FIELD(switch_event, prev_state, event, raw->data);
1364         FILL_ARRAY(switch_event, next_comm, event, raw->data);
1365         FILL_FIELD(switch_event, next_pid, event, raw->data);
1366         FILL_FIELD(switch_event, next_prio, event, raw->data);
1367
1368         trace_handler->switch_event(&switch_event, event, cpu, timestamp, thread);
1369 }
1370
1371 static void
1372 process_sched_fork_event(struct raw_event_sample *raw,
1373                          struct event *event,
1374                          int cpu __used,
1375                          u64 timestamp __used,
1376                          struct thread *thread __used)
1377 {
1378         struct trace_fork_event fork_event;
1379
1380         FILL_COMMON_FIELDS(fork_event, event, raw->data);
1381
1382         FILL_ARRAY(fork_event, parent_comm, event, raw->data);
1383         FILL_FIELD(fork_event, parent_pid, event, raw->data);
1384         FILL_ARRAY(fork_event, child_comm, event, raw->data);
1385         FILL_FIELD(fork_event, child_pid, event, raw->data);
1386
1387         trace_handler->fork_event(&fork_event, event, cpu, timestamp, thread);
1388 }
1389
1390 static void
1391 process_sched_exit_event(struct event *event,
1392                          int cpu __used,
1393                          u64 timestamp __used,
1394                          struct thread *thread __used)
1395 {
1396         if (verbose)
1397                 printf("sched_exit event %p\n", event);
1398 }
1399
1400 static void
1401 process_raw_event(event_t *raw_event __used, void *more_data,
1402                   int cpu, u64 timestamp, struct thread *thread)
1403 {
1404         struct raw_event_sample *raw = more_data;
1405         struct event *event;
1406         int type;
1407
1408         type = trace_parse_common_type(raw->data);
1409         event = trace_find_event(type);
1410
1411         if (!strcmp(event->name, "sched_switch"))
1412                 process_sched_switch_event(raw, event, cpu, timestamp, thread);
1413         if (!strcmp(event->name, "sched_wakeup"))
1414                 process_sched_wakeup_event(raw, event, cpu, timestamp, thread);
1415         if (!strcmp(event->name, "sched_wakeup_new"))
1416                 process_sched_wakeup_event(raw, event, cpu, timestamp, thread);
1417         if (!strcmp(event->name, "sched_process_fork"))
1418                 process_sched_fork_event(raw, event, cpu, timestamp, thread);
1419         if (!strcmp(event->name, "sched_process_exit"))
1420                 process_sched_exit_event(event, cpu, timestamp, thread);
1421 }
1422
1423 static int
1424 process_sample_event(event_t *event, unsigned long offset, unsigned long head)
1425 {
1426         char level;
1427         int show = 0;
1428         struct dso *dso = NULL;
1429         struct thread *thread;
1430         u64 ip = event->ip.ip;
1431         u64 timestamp = -1;
1432         u32 cpu = -1;
1433         u64 period = 1;
1434         void *more_data = event->ip.__more_data;
1435         int cpumode;
1436
1437         thread = threads__findnew(event->ip.pid, &threads, &last_match);
1438
1439         if (sample_type & PERF_SAMPLE_TIME) {
1440                 timestamp = *(u64 *)more_data;
1441                 more_data += sizeof(u64);
1442         }
1443
1444         if (sample_type & PERF_SAMPLE_CPU) {
1445                 cpu = *(u32 *)more_data;
1446                 more_data += sizeof(u32);
1447                 more_data += sizeof(u32); /* reserved */
1448         }
1449
1450         if (sample_type & PERF_SAMPLE_PERIOD) {
1451                 period = *(u64 *)more_data;
1452                 more_data += sizeof(u64);
1453         }
1454
1455         dump_printf("%p [%p]: PERF_EVENT_SAMPLE (IP, %d): %d/%d: %p period: %Ld\n",
1456                 (void *)(offset + head),
1457                 (void *)(long)(event->header.size),
1458                 event->header.misc,
1459                 event->ip.pid, event->ip.tid,
1460                 (void *)(long)ip,
1461                 (long long)period);
1462
1463         dump_printf(" ... thread: %s:%d\n", thread->comm, thread->pid);
1464
1465         if (thread == NULL) {
1466                 eprintf("problem processing %d event, skipping it.\n",
1467                         event->header.type);
1468                 return -1;
1469         }
1470
1471         cpumode = event->header.misc & PERF_EVENT_MISC_CPUMODE_MASK;
1472
1473         if (cpumode == PERF_EVENT_MISC_KERNEL) {
1474                 show = SHOW_KERNEL;
1475                 level = 'k';
1476
1477                 dso = kernel_dso;
1478
1479                 dump_printf(" ...... dso: %s\n", dso->name);
1480
1481         } else if (cpumode == PERF_EVENT_MISC_USER) {
1482
1483                 show = SHOW_USER;
1484                 level = '.';
1485
1486         } else {
1487                 show = SHOW_HV;
1488                 level = 'H';
1489
1490                 dso = hypervisor_dso;
1491
1492                 dump_printf(" ...... dso: [hypervisor]\n");
1493         }
1494
1495         if (sample_type & PERF_SAMPLE_RAW)
1496                 process_raw_event(event, more_data, cpu, timestamp, thread);
1497
1498         return 0;
1499 }
1500
1501 static int
1502 process_event(event_t *event, unsigned long offset, unsigned long head)
1503 {
1504         trace_event(event);
1505
1506         switch (event->header.type) {
1507         case PERF_EVENT_MMAP ... PERF_EVENT_LOST:
1508                 return 0;
1509
1510         case PERF_EVENT_COMM:
1511                 return process_comm_event(event, offset, head);
1512
1513         case PERF_EVENT_EXIT ... PERF_EVENT_READ:
1514                 return 0;
1515
1516         case PERF_EVENT_SAMPLE:
1517                 return process_sample_event(event, offset, head);
1518
1519         case PERF_EVENT_MAX:
1520         default:
1521                 return -1;
1522         }
1523
1524         return 0;
1525 }
1526
1527 static int read_events(void)
1528 {
1529         int ret, rc = EXIT_FAILURE;
1530         unsigned long offset = 0;
1531         unsigned long head = 0;
1532         struct stat perf_stat;
1533         event_t *event;
1534         uint32_t size;
1535         char *buf;
1536
1537         trace_report();
1538         register_idle_thread(&threads, &last_match);
1539
1540         input = open(input_name, O_RDONLY);
1541         if (input < 0) {
1542                 perror("failed to open file");
1543                 exit(-1);
1544         }
1545
1546         ret = fstat(input, &perf_stat);
1547         if (ret < 0) {
1548                 perror("failed to stat file");
1549                 exit(-1);
1550         }
1551
1552         if (!perf_stat.st_size) {
1553                 fprintf(stderr, "zero-sized file, nothing to do!\n");
1554                 exit(0);
1555         }
1556         header = perf_header__read(input);
1557         head = header->data_offset;
1558         sample_type = perf_header__sample_type(header);
1559
1560         if (!(sample_type & PERF_SAMPLE_RAW))
1561                 die("No trace sample to read. Did you call perf record "
1562                     "without -R?");
1563
1564         if (load_kernel() < 0) {
1565                 perror("failed to load kernel symbols");
1566                 return EXIT_FAILURE;
1567         }
1568
1569 remap:
1570         buf = (char *)mmap(NULL, page_size * mmap_window, PROT_READ,
1571                            MAP_SHARED, input, offset);
1572         if (buf == MAP_FAILED) {
1573                 perror("failed to mmap file");
1574                 exit(-1);
1575         }
1576
1577 more:
1578         event = (event_t *)(buf + head);
1579
1580         size = event->header.size;
1581         if (!size)
1582                 size = 8;
1583
1584         if (head + event->header.size >= page_size * mmap_window) {
1585                 unsigned long shift = page_size * (head / page_size);
1586                 int res;
1587
1588                 res = munmap(buf, page_size * mmap_window);
1589                 assert(res == 0);
1590
1591                 offset += shift;
1592                 head -= shift;
1593                 goto remap;
1594         }
1595
1596         size = event->header.size;
1597
1598
1599         if (!size || process_event(event, offset, head) < 0) {
1600
1601                 /*
1602                  * assume we lost track of the stream, check alignment, and
1603                  * increment a single u64 in the hope to catch on again 'soon'.
1604                  */
1605
1606                 if (unlikely(head & 7))
1607                         head &= ~7ULL;
1608
1609                 size = 8;
1610         }
1611
1612         head += size;
1613
1614         if (offset + head < (unsigned long)perf_stat.st_size)
1615                 goto more;
1616
1617         rc = EXIT_SUCCESS;
1618         close(input);
1619
1620         return rc;
1621 }
1622
1623 static const char * const sched_usage[] = {
1624         "perf sched [<options>] {record|latency|replay|trace}",
1625         NULL
1626 };
1627
1628 static const struct option sched_options[] = {
1629         OPT_BOOLEAN('v', "verbose", &verbose,
1630                     "be more verbose (show symbol address, etc)"),
1631         OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
1632                     "dump raw trace in ASCII"),
1633         OPT_END()
1634 };
1635
1636 static const char * const latency_usage[] = {
1637         "perf sched latency [<options>]",
1638         NULL
1639 };
1640
1641 static const struct option latency_options[] = {
1642         OPT_STRING('s', "sort", &sort_order, "key[,key2...]",
1643                    "sort by key(s): runtime, switch, avg, max"),
1644         OPT_BOOLEAN('v', "verbose", &verbose,
1645                     "be more verbose (show symbol address, etc)"),
1646         OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
1647                     "dump raw trace in ASCII"),
1648         OPT_END()
1649 };
1650
1651 static const char * const replay_usage[] = {
1652         "perf sched replay [<options>]",
1653         NULL
1654 };
1655
1656 static const struct option replay_options[] = {
1657         OPT_INTEGER('r', "repeat", &replay_repeat,
1658                     "repeat the workload replay N times (-1: infinite)"),
1659         OPT_BOOLEAN('v', "verbose", &verbose,
1660                     "be more verbose (show symbol address, etc)"),
1661         OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
1662                     "dump raw trace in ASCII"),
1663         OPT_END()
1664 };
1665
1666 static void setup_sorting(void)
1667 {
1668         char *tmp, *tok, *str = strdup(sort_order);
1669
1670         for (tok = strtok_r(str, ", ", &tmp);
1671                         tok; tok = strtok_r(NULL, ", ", &tmp)) {
1672                 if (sort_dimension__add(tok, &sort_list) < 0) {
1673                         error("Unknown --sort key: `%s'", tok);
1674                         usage_with_options(latency_usage, latency_options);
1675                 }
1676         }
1677
1678         free(str);
1679
1680         sort_dimension__add((char *)"pid", &cmp_pid);
1681 }
1682
1683 static const char *record_args[] = {
1684         "record",
1685         "-a",
1686         "-R",
1687         "-M",
1688         "-f",
1689         "-c", "1",
1690         "-e", "sched:sched_switch:r",
1691         "-e", "sched:sched_stat_wait:r",
1692         "-e", "sched:sched_stat_sleep:r",
1693         "-e", "sched:sched_stat_iowait:r",
1694         "-e", "sched:sched_stat_runtime:r",
1695         "-e", "sched:sched_process_exit:r",
1696         "-e", "sched:sched_process_fork:r",
1697         "-e", "sched:sched_wakeup:r",
1698         "-e", "sched:sched_migrate_task:r",
1699 };
1700
1701 static int __cmd_record(int argc, const char **argv)
1702 {
1703         unsigned int rec_argc, i, j;
1704         const char **rec_argv;
1705
1706         rec_argc = ARRAY_SIZE(record_args) + argc - 1;
1707         rec_argv = calloc(rec_argc + 1, sizeof(char *));
1708
1709         for (i = 0; i < ARRAY_SIZE(record_args); i++)
1710                 rec_argv[i] = strdup(record_args[i]);
1711
1712         for (j = 1; j < (unsigned int)argc; j++, i++)
1713                 rec_argv[i] = argv[j];
1714
1715         BUG_ON(i != rec_argc);
1716
1717         return cmd_record(i, rec_argv, NULL);
1718 }
1719
1720 int cmd_sched(int argc, const char **argv, const char *prefix __used)
1721 {
1722         symbol__init();
1723         page_size = getpagesize();
1724
1725         argc = parse_options(argc, argv, sched_options, sched_usage,
1726                              PARSE_OPT_STOP_AT_NON_OPTION);
1727         if (!argc)
1728                 usage_with_options(sched_usage, sched_options);
1729
1730         if (!strncmp(argv[0], "rec", 3)) {
1731                 return __cmd_record(argc, argv);
1732         } else if (!strncmp(argv[0], "lat", 3)) {
1733                 trace_handler = &lat_ops;
1734                 if (argc > 1) {
1735                         argc = parse_options(argc, argv, latency_options, latency_usage, 0);
1736                         if (argc)
1737                                 usage_with_options(latency_usage, latency_options);
1738                 }
1739                 setup_sorting();
1740                 __cmd_lat();
1741         } else if (!strncmp(argv[0], "rep", 3)) {
1742                 trace_handler = &replay_ops;
1743                 if (argc) {
1744                         argc = parse_options(argc, argv, replay_options, replay_usage, 0);
1745                         if (argc)
1746                                 usage_with_options(replay_usage, replay_options);
1747                 }
1748                 __cmd_replay();
1749         } else if (!strcmp(argv[0], "trace")) {
1750                 /*
1751                  * Aliased to 'perf trace' for now:
1752                  */
1753                 return cmd_trace(argc, argv, prefix);
1754         } else {
1755                 usage_with_options(sched_usage, sched_options);
1756         }
1757
1758         return 0;
1759 }