1
2
3
4
5
6
7#include <linux/kernel.h>
8#include <linux/sched.h>
9#include <linux/sched/clock.h>
10#include <linux/perf_event.h>
11#include <linux/percpu.h>
12#include <linux/hardirq.h>
13#include <linux/uaccess.h>
14#include <asm/reg.h>
15#include <asm/pmc.h>
16#include <asm/machdep.h>
17#include <asm/firmware.h>
18#include <asm/ptrace.h>
19#include <asm/code-patching.h>
20#include <asm/interrupt.h>
21
22#ifdef CONFIG_PPC64
23#include "internal.h"
24#endif
25
26#define BHRB_MAX_ENTRIES 32
27#define BHRB_TARGET 0x0000000000000002
28#define BHRB_PREDICTION 0x0000000000000001
29#define BHRB_EA 0xFFFFFFFFFFFFFFFCUL
30
31struct cpu_hw_events {
32 int n_events;
33 int n_percpu;
34 int disabled;
35 int n_added;
36 int n_limited;
37 u8 pmcs_enabled;
38 struct perf_event *event[MAX_HWEVENTS];
39 u64 events[MAX_HWEVENTS];
40 unsigned int flags[MAX_HWEVENTS];
41 struct mmcr_regs mmcr;
42 struct perf_event *limited_counter[MAX_LIMITED_HWCOUNTERS];
43 u8 limited_hwidx[MAX_LIMITED_HWCOUNTERS];
44 u64 alternatives[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
45 unsigned long amasks[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
46 unsigned long avalues[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
47
48 unsigned int txn_flags;
49 int n_txn_start;
50
51
52 u64 bhrb_filter;
53 unsigned int bhrb_users;
54 void *bhrb_context;
55 struct perf_branch_stack bhrb_stack;
56 struct perf_branch_entry bhrb_entries[BHRB_MAX_ENTRIES];
57 u64 ic_init;
58
59
60 unsigned long pmcs[MAX_HWEVENTS];
61};
62
63static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
64
65static struct power_pmu *ppmu;
66
67
68
69
70
71
72
73
74static unsigned int freeze_events_kernel = MMCR0_FCS;
75
76
77
78
79
80
81
82
83
84
85#ifdef CONFIG_PPC32
86
87#define MMCR0_FCHV 0
88#define MMCR0_PMCjCE MMCR0_PMCnCE
89#define MMCR0_FC56 0
90#define MMCR0_PMAO 0
91#define MMCR0_EBE 0
92#define MMCR0_BHRBA 0
93#define MMCR0_PMCC 0
94#define MMCR0_PMCC_U6 0
95
96#define SPRN_MMCRA SPRN_MMCR2
97#define SPRN_MMCR3 0
98#define SPRN_SIER2 0
99#define SPRN_SIER3 0
100#define MMCRA_SAMPLE_ENABLE 0
101#define MMCRA_BHRB_DISABLE 0
102#define MMCR0_PMCCEXT 0
103
104static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
105{
106 return 0;
107}
108static inline void perf_get_data_addr(struct perf_event *event, struct pt_regs *regs, u64 *addrp) { }
109static inline u32 perf_get_misc_flags(struct pt_regs *regs)
110{
111 return 0;
112}
113static inline void perf_read_regs(struct pt_regs *regs)
114{
115 regs->result = 0;
116}
117
118static inline int siar_valid(struct pt_regs *regs)
119{
120 return 1;
121}
122
123static bool is_ebb_event(struct perf_event *event) { return false; }
124static int ebb_event_check(struct perf_event *event) { return 0; }
125static void ebb_event_add(struct perf_event *event) { }
126static void ebb_switch_out(unsigned long mmcr0) { }
127static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw)
128{
129 return cpuhw->mmcr.mmcr0;
130}
131
132static inline void power_pmu_bhrb_enable(struct perf_event *event) {}
133static inline void power_pmu_bhrb_disable(struct perf_event *event) {}
134static void power_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) {}
135static inline void power_pmu_bhrb_read(struct perf_event *event, struct cpu_hw_events *cpuhw) {}
136static void pmao_restore_workaround(bool ebb) { }
137#endif
138
139bool is_sier_available(void)
140{
141 if (!ppmu)
142 return false;
143
144 if (ppmu->flags & PPMU_HAS_SIER)
145 return true;
146
147 return false;
148}
149
150
151
152
153
154unsigned long get_pmcs_ext_regs(int idx)
155{
156 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
157
158 return cpuhw->pmcs[idx];
159}
160
161static bool regs_use_siar(struct pt_regs *regs)
162{
163
164
165
166
167
168
169
170
171
172 return ((TRAP(regs) == INTERRUPT_PERFMON) && regs->result);
173}
174
175
176
177
178#ifdef CONFIG_PPC64
179
180static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
181{
182 unsigned long mmcra = regs->dsisr;
183
184 if ((ppmu->flags & PPMU_HAS_SSLOT) && (mmcra & MMCRA_SAMPLE_ENABLE)) {
185 unsigned long slot = (mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT;
186 if (slot > 1)
187 return 4 * (slot - 1);
188 }
189
190 return 0;
191}
192
193
194
195
196
197
198
199
200
201static inline void perf_get_data_addr(struct perf_event *event, struct pt_regs *regs, u64 *addrp)
202{
203 unsigned long mmcra = regs->dsisr;
204 bool sdar_valid;
205
206 if (ppmu->flags & PPMU_HAS_SIER)
207 sdar_valid = regs->dar & SIER_SDAR_VALID;
208 else {
209 unsigned long sdsync;
210
211 if (ppmu->flags & PPMU_SIAR_VALID)
212 sdsync = POWER7P_MMCRA_SDAR_VALID;
213 else if (ppmu->flags & PPMU_ALT_SIPR)
214 sdsync = POWER6_MMCRA_SDSYNC;
215 else if (ppmu->flags & PPMU_NO_SIAR)
216 sdsync = MMCRA_SAMPLE_ENABLE;
217 else
218 sdsync = MMCRA_SDSYNC;
219
220 sdar_valid = mmcra & sdsync;
221 }
222
223 if (!(mmcra & MMCRA_SAMPLE_ENABLE) || sdar_valid)
224 *addrp = mfspr(SPRN_SDAR);
225
226 if (is_kernel_addr(mfspr(SPRN_SDAR)) && event->attr.exclude_kernel)
227 *addrp = 0;
228}
229
230static bool regs_sihv(struct pt_regs *regs)
231{
232 unsigned long sihv = MMCRA_SIHV;
233
234 if (ppmu->flags & PPMU_HAS_SIER)
235 return !!(regs->dar & SIER_SIHV);
236
237 if (ppmu->flags & PPMU_ALT_SIPR)
238 sihv = POWER6_MMCRA_SIHV;
239
240 return !!(regs->dsisr & sihv);
241}
242
243static bool regs_sipr(struct pt_regs *regs)
244{
245 unsigned long sipr = MMCRA_SIPR;
246
247 if (ppmu->flags & PPMU_HAS_SIER)
248 return !!(regs->dar & SIER_SIPR);
249
250 if (ppmu->flags & PPMU_ALT_SIPR)
251 sipr = POWER6_MMCRA_SIPR;
252
253 return !!(regs->dsisr & sipr);
254}
255
256static inline u32 perf_flags_from_msr(struct pt_regs *regs)
257{
258 if (regs->msr & MSR_PR)
259 return PERF_RECORD_MISC_USER;
260 if ((regs->msr & MSR_HV) && freeze_events_kernel != MMCR0_FCHV)
261 return PERF_RECORD_MISC_HYPERVISOR;
262 return PERF_RECORD_MISC_KERNEL;
263}
264
265static inline u32 perf_get_misc_flags(struct pt_regs *regs)
266{
267 bool use_siar = regs_use_siar(regs);
268 unsigned long mmcra = regs->dsisr;
269 int marked = mmcra & MMCRA_SAMPLE_ENABLE;
270
271 if (!use_siar)
272 return perf_flags_from_msr(regs);
273
274
275
276
277
278
279
280 if (marked && (ppmu->flags & PPMU_P10_DD1)) {
281 unsigned long siar = mfspr(SPRN_SIAR);
282 if (siar) {
283 if (is_kernel_addr(siar))
284 return PERF_RECORD_MISC_KERNEL;
285 return PERF_RECORD_MISC_USER;
286 } else {
287 if (is_kernel_addr(regs->nip))
288 return PERF_RECORD_MISC_KERNEL;
289 return PERF_RECORD_MISC_USER;
290 }
291 }
292
293
294
295
296
297
298
299 if (ppmu->flags & PPMU_NO_SIPR) {
300 unsigned long siar = mfspr(SPRN_SIAR);
301 if (is_kernel_addr(siar))
302 return PERF_RECORD_MISC_KERNEL;
303 return PERF_RECORD_MISC_USER;
304 }
305
306
307 if (regs_sipr(regs))
308 return PERF_RECORD_MISC_USER;
309
310 if (regs_sihv(regs) && (freeze_events_kernel != MMCR0_FCHV))
311 return PERF_RECORD_MISC_HYPERVISOR;
312
313 return PERF_RECORD_MISC_KERNEL;
314}
315
316
317
318
319
320
321
322
323static inline void perf_read_regs(struct pt_regs *regs)
324{
325 unsigned long mmcra = mfspr(SPRN_MMCRA);
326 int marked = mmcra & MMCRA_SAMPLE_ENABLE;
327 int use_siar;
328
329 regs->dsisr = mmcra;
330
331 if (ppmu->flags & PPMU_HAS_SIER)
332 regs->dar = mfspr(SPRN_SIER);
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351 if (TRAP(regs) != INTERRUPT_PERFMON)
352 use_siar = 0;
353 else if ((ppmu->flags & PPMU_NO_SIAR))
354 use_siar = 0;
355 else if (marked)
356 use_siar = 1;
357 else if ((ppmu->flags & PPMU_NO_CONT_SAMPLING))
358 use_siar = 0;
359 else if (!(ppmu->flags & PPMU_NO_SIPR) && regs_sipr(regs))
360 use_siar = 0;
361 else
362 use_siar = 1;
363
364 regs->result = use_siar;
365}
366
367
368
369
370
371
372
373
374static inline int siar_valid(struct pt_regs *regs)
375{
376 unsigned long mmcra = regs->dsisr;
377 int marked = mmcra & MMCRA_SAMPLE_ENABLE;
378
379 if (marked) {
380
381
382
383
384
385 if (ppmu->flags & PPMU_P10_DD1)
386 return 0x1;
387 else if (ppmu->flags & PPMU_HAS_SIER)
388 return regs->dar & SIER_SIAR_VALID;
389
390 if (ppmu->flags & PPMU_SIAR_VALID)
391 return mmcra & POWER7P_MMCRA_SIAR_VALID;
392 }
393
394 return 1;
395}
396
397
398
399static void power_pmu_bhrb_reset(void)
400{
401 asm volatile(PPC_CLRBHRB);
402}
403
404static void power_pmu_bhrb_enable(struct perf_event *event)
405{
406 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
407
408 if (!ppmu->bhrb_nr)
409 return;
410
411
412 if (event->ctx->task && cpuhw->bhrb_context != event->ctx) {
413 power_pmu_bhrb_reset();
414 cpuhw->bhrb_context = event->ctx;
415 }
416 cpuhw->bhrb_users++;
417 perf_sched_cb_inc(event->ctx->pmu);
418}
419
420static void power_pmu_bhrb_disable(struct perf_event *event)
421{
422 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
423
424 if (!ppmu->bhrb_nr)
425 return;
426
427 WARN_ON_ONCE(!cpuhw->bhrb_users);
428 cpuhw->bhrb_users--;
429 perf_sched_cb_dec(event->ctx->pmu);
430
431 if (!cpuhw->disabled && !cpuhw->bhrb_users) {
432
433
434
435
436
437 cpuhw->bhrb_context = NULL;
438 }
439}
440
441
442
443
444static void power_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
445{
446 if (!ppmu->bhrb_nr)
447 return;
448
449 if (sched_in)
450 power_pmu_bhrb_reset();
451}
452
453static __u64 power_pmu_bhrb_to(u64 addr)
454{
455 unsigned int instr;
456 __u64 target;
457
458 if (is_kernel_addr(addr)) {
459 if (copy_from_kernel_nofault(&instr, (void *)addr,
460 sizeof(instr)))
461 return 0;
462
463 return branch_target(&instr);
464 }
465
466
467 if (copy_from_user_nofault(&instr, (unsigned int __user *)addr,
468 sizeof(instr)))
469 return 0;
470
471 target = branch_target(&instr);
472 if ((!target) || (instr & BRANCH_ABSOLUTE))
473 return target;
474
475
476 return target - (unsigned long)&instr + addr;
477}
478
479
480static void power_pmu_bhrb_read(struct perf_event *event, struct cpu_hw_events *cpuhw)
481{
482 u64 val;
483 u64 addr;
484 int r_index, u_index, pred;
485
486 r_index = 0;
487 u_index = 0;
488 while (r_index < ppmu->bhrb_nr) {
489
490 val = read_bhrb(r_index++);
491 if (!val)
492
493 break;
494 else {
495 addr = val & BHRB_EA;
496 pred = val & BHRB_PREDICTION;
497
498 if (!addr)
499
500 continue;
501
502
503
504
505
506
507
508
509
510 if (!(ppmu->flags & PPMU_ARCH_31) &&
511 is_kernel_addr(addr) && event->attr.exclude_kernel)
512 continue;
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532 if (val & BHRB_TARGET) {
533
534
535
536 cpuhw->bhrb_entries[u_index].to = addr;
537 cpuhw->bhrb_entries[u_index].mispred = pred;
538 cpuhw->bhrb_entries[u_index].predicted = ~pred;
539
540
541 val = read_bhrb(r_index++);
542 addr = val & BHRB_EA;
543 if (val & BHRB_TARGET) {
544
545
546 r_index--;
547 addr = 0;
548 }
549 cpuhw->bhrb_entries[u_index].from = addr;
550 } else {
551
552
553 cpuhw->bhrb_entries[u_index].from = addr;
554 cpuhw->bhrb_entries[u_index].to =
555 power_pmu_bhrb_to(addr);
556 cpuhw->bhrb_entries[u_index].mispred = pred;
557 cpuhw->bhrb_entries[u_index].predicted = ~pred;
558 }
559 u_index++;
560
561 }
562 }
563 cpuhw->bhrb_stack.nr = u_index;
564 cpuhw->bhrb_stack.hw_idx = -1ULL;
565 return;
566}
567
568static bool is_ebb_event(struct perf_event *event)
569{
570
571
572
573
574
575 return (ppmu->flags & PPMU_ARCH_207S) &&
576 ((event->attr.config >> PERF_EVENT_CONFIG_EBB_SHIFT) & 1);
577}
578
579static int ebb_event_check(struct perf_event *event)
580{
581 struct perf_event *leader = event->group_leader;
582
583
584 if (is_ebb_event(leader) != is_ebb_event(event))
585 return -EINVAL;
586
587 if (is_ebb_event(event)) {
588 if (!(event->attach_state & PERF_ATTACH_TASK))
589 return -EINVAL;
590
591 if (!leader->attr.pinned || !leader->attr.exclusive)
592 return -EINVAL;
593
594 if (event->attr.freq ||
595 event->attr.inherit ||
596 event->attr.sample_type ||
597 event->attr.sample_period ||
598 event->attr.enable_on_exec)
599 return -EINVAL;
600 }
601
602 return 0;
603}
604
605static void ebb_event_add(struct perf_event *event)
606{
607 if (!is_ebb_event(event) || current->thread.used_ebb)
608 return;
609
610
611
612
613
614
615
616 current->thread.used_ebb = 1;
617 current->thread.mmcr0 |= MMCR0_PMXE;
618}
619
620static void ebb_switch_out(unsigned long mmcr0)
621{
622 if (!(mmcr0 & MMCR0_EBE))
623 return;
624
625 current->thread.siar = mfspr(SPRN_SIAR);
626 current->thread.sier = mfspr(SPRN_SIER);
627 current->thread.sdar = mfspr(SPRN_SDAR);
628 current->thread.mmcr0 = mmcr0 & MMCR0_USER_MASK;
629 current->thread.mmcr2 = mfspr(SPRN_MMCR2) & MMCR2_USER_MASK;
630 if (ppmu->flags & PPMU_ARCH_31) {
631 current->thread.mmcr3 = mfspr(SPRN_MMCR3);
632 current->thread.sier2 = mfspr(SPRN_SIER2);
633 current->thread.sier3 = mfspr(SPRN_SIER3);
634 }
635}
636
637static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw)
638{
639 unsigned long mmcr0 = cpuhw->mmcr.mmcr0;
640
641 if (!ebb)
642 goto out;
643
644
645 mmcr0 |= MMCR0_EBE | MMCR0_BHRBA | MMCR0_PMCC_U6;
646
647
648
649
650
651
652 mmcr0 |= current->thread.mmcr0;
653
654
655
656
657
658
659 if (!(current->thread.mmcr0 & MMCR0_PMXE))
660 mmcr0 &= ~MMCR0_PMXE;
661
662 mtspr(SPRN_SIAR, current->thread.siar);
663 mtspr(SPRN_SIER, current->thread.sier);
664 mtspr(SPRN_SDAR, current->thread.sdar);
665
666
667
668
669
670
671
672
673 mtspr(SPRN_MMCR2, cpuhw->mmcr.mmcr2 | current->thread.mmcr2);
674
675 if (ppmu->flags & PPMU_ARCH_31) {
676 mtspr(SPRN_MMCR3, current->thread.mmcr3);
677 mtspr(SPRN_SIER2, current->thread.sier2);
678 mtspr(SPRN_SIER3, current->thread.sier3);
679 }
680out:
681 return mmcr0;
682}
683
684static void pmao_restore_workaround(bool ebb)
685{
686 unsigned pmcs[6];
687
688 if (!cpu_has_feature(CPU_FTR_PMAO_BUG))
689 return;
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724 if ((current->thread.mmcr0 & (MMCR0_PMAO | MMCR0_PMAO_SYNC)) != MMCR0_PMAO)
725 return;
726
727
728 if (ebb && !(current->thread.bescr & BESCR_GE))
729 return;
730
731
732
733
734
735 hard_irq_disable();
736
737
738
739
740
741
742 pmcs[0] = mfspr(SPRN_PMC1);
743 pmcs[1] = mfspr(SPRN_PMC2);
744 pmcs[2] = mfspr(SPRN_PMC3);
745 pmcs[3] = mfspr(SPRN_PMC4);
746 pmcs[4] = mfspr(SPRN_PMC5);
747 pmcs[5] = mfspr(SPRN_PMC6);
748
749
750 mtspr(SPRN_MMCR2, 0);
751
752
753 mtspr(SPRN_PMC6, 0x7FFFFFFE);
754
755
756 mtspr(SPRN_MMCR0, MMCR0_PMXE | MMCR0_PMCjCE | MMCR0_PMAO);
757
758
759 mtspr(SPRN_MMCR0, MMCR0_FC | MMCR0_PMAO);
760
761 mtspr(SPRN_PMC1, pmcs[0]);
762 mtspr(SPRN_PMC2, pmcs[1]);
763 mtspr(SPRN_PMC3, pmcs[2]);
764 mtspr(SPRN_PMC4, pmcs[3]);
765 mtspr(SPRN_PMC5, pmcs[4]);
766 mtspr(SPRN_PMC6, pmcs[5]);
767}
768
769#endif
770
771static void perf_event_interrupt(struct pt_regs *regs);
772
773
774
775
776static unsigned long read_pmc(int idx)
777{
778 unsigned long val;
779
780 switch (idx) {
781 case 1:
782 val = mfspr(SPRN_PMC1);
783 break;
784 case 2:
785 val = mfspr(SPRN_PMC2);
786 break;
787 case 3:
788 val = mfspr(SPRN_PMC3);
789 break;
790 case 4:
791 val = mfspr(SPRN_PMC4);
792 break;
793 case 5:
794 val = mfspr(SPRN_PMC5);
795 break;
796 case 6:
797 val = mfspr(SPRN_PMC6);
798 break;
799#ifdef CONFIG_PPC64
800 case 7:
801 val = mfspr(SPRN_PMC7);
802 break;
803 case 8:
804 val = mfspr(SPRN_PMC8);
805 break;
806#endif
807 default:
808 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
809 val = 0;
810 }
811 return val;
812}
813
814
815
816
817static void write_pmc(int idx, unsigned long val)
818{
819 switch (idx) {
820 case 1:
821 mtspr(SPRN_PMC1, val);
822 break;
823 case 2:
824 mtspr(SPRN_PMC2, val);
825 break;
826 case 3:
827 mtspr(SPRN_PMC3, val);
828 break;
829 case 4:
830 mtspr(SPRN_PMC4, val);
831 break;
832 case 5:
833 mtspr(SPRN_PMC5, val);
834 break;
835 case 6:
836 mtspr(SPRN_PMC6, val);
837 break;
838#ifdef CONFIG_PPC64
839 case 7:
840 mtspr(SPRN_PMC7, val);
841 break;
842 case 8:
843 mtspr(SPRN_PMC8, val);
844 break;
845#endif
846 default:
847 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
848 }
849}
850
851
852void perf_event_print_debug(void)
853{
854 unsigned long sdar, sier, flags;
855 u32 pmcs[MAX_HWEVENTS];
856 int i;
857
858 if (!ppmu) {
859 pr_info("Performance monitor hardware not registered.\n");
860 return;
861 }
862
863 if (!ppmu->n_counter)
864 return;
865
866 local_irq_save(flags);
867
868 pr_info("CPU: %d PMU registers, ppmu = %s n_counters = %d",
869 smp_processor_id(), ppmu->name, ppmu->n_counter);
870
871 for (i = 0; i < ppmu->n_counter; i++)
872 pmcs[i] = read_pmc(i + 1);
873
874 for (; i < MAX_HWEVENTS; i++)
875 pmcs[i] = 0xdeadbeef;
876
877 pr_info("PMC1: %08x PMC2: %08x PMC3: %08x PMC4: %08x\n",
878 pmcs[0], pmcs[1], pmcs[2], pmcs[3]);
879
880 if (ppmu->n_counter > 4)
881 pr_info("PMC5: %08x PMC6: %08x PMC7: %08x PMC8: %08x\n",
882 pmcs[4], pmcs[5], pmcs[6], pmcs[7]);
883
884 pr_info("MMCR0: %016lx MMCR1: %016lx MMCRA: %016lx\n",
885 mfspr(SPRN_MMCR0), mfspr(SPRN_MMCR1), mfspr(SPRN_MMCRA));
886
887 sdar = sier = 0;
888#ifdef CONFIG_PPC64
889 sdar = mfspr(SPRN_SDAR);
890
891 if (ppmu->flags & PPMU_HAS_SIER)
892 sier = mfspr(SPRN_SIER);
893
894 if (ppmu->flags & PPMU_ARCH_207S) {
895 pr_info("MMCR2: %016lx EBBHR: %016lx\n",
896 mfspr(SPRN_MMCR2), mfspr(SPRN_EBBHR));
897 pr_info("EBBRR: %016lx BESCR: %016lx\n",
898 mfspr(SPRN_EBBRR), mfspr(SPRN_BESCR));
899 }
900
901 if (ppmu->flags & PPMU_ARCH_31) {
902 pr_info("MMCR3: %016lx SIER2: %016lx SIER3: %016lx\n",
903 mfspr(SPRN_MMCR3), mfspr(SPRN_SIER2), mfspr(SPRN_SIER3));
904 }
905#endif
906 pr_info("SIAR: %016lx SDAR: %016lx SIER: %016lx\n",
907 mfspr(SPRN_SIAR), sdar, sier);
908
909 local_irq_restore(flags);
910}
911
912
913
914
915
916
917
918static int power_check_constraints(struct cpu_hw_events *cpuhw,
919 u64 event_id[], unsigned int cflags[],
920 int n_ev, struct perf_event **event)
921{
922 unsigned long mask, value, nv;
923 unsigned long smasks[MAX_HWEVENTS], svalues[MAX_HWEVENTS];
924 int n_alt[MAX_HWEVENTS], choice[MAX_HWEVENTS];
925 int i, j;
926 unsigned long addf = ppmu->add_fields;
927 unsigned long tadd = ppmu->test_adder;
928 unsigned long grp_mask = ppmu->group_constraint_mask;
929 unsigned long grp_val = ppmu->group_constraint_val;
930
931 if (n_ev > ppmu->n_counter)
932 return -1;
933
934
935 for (i = 0; i < n_ev; ++i) {
936 if ((cflags[i] & PPMU_LIMITED_PMC_REQD)
937 && !ppmu->limited_pmc_event(event_id[i])) {
938 ppmu->get_alternatives(event_id[i], cflags[i],
939 cpuhw->alternatives[i]);
940 event_id[i] = cpuhw->alternatives[i][0];
941 }
942 if (ppmu->get_constraint(event_id[i], &cpuhw->amasks[i][0],
943 &cpuhw->avalues[i][0], event[i]->attr.config1))
944 return -1;
945 }
946 value = mask = 0;
947 for (i = 0; i < n_ev; ++i) {
948 nv = (value | cpuhw->avalues[i][0]) +
949 (value & cpuhw->avalues[i][0] & addf);
950
951 if (((((nv + tadd) ^ value) & mask) & (~grp_mask)) != 0)
952 break;
953
954 if (((((nv + tadd) ^ cpuhw->avalues[i][0]) & cpuhw->amasks[i][0])
955 & (~grp_mask)) != 0)
956 break;
957
958 value = nv;
959 mask |= cpuhw->amasks[i][0];
960 }
961 if (i == n_ev) {
962 if ((value & mask & grp_mask) != (mask & grp_val))
963 return -1;
964 else
965 return 0;
966 }
967
968
969 if (!ppmu->get_alternatives)
970 return -1;
971 for (i = 0; i < n_ev; ++i) {
972 choice[i] = 0;
973 n_alt[i] = ppmu->get_alternatives(event_id[i], cflags[i],
974 cpuhw->alternatives[i]);
975 for (j = 1; j < n_alt[i]; ++j)
976 ppmu->get_constraint(cpuhw->alternatives[i][j],
977 &cpuhw->amasks[i][j],
978 &cpuhw->avalues[i][j],
979 event[i]->attr.config1);
980 }
981
982
983 i = 0;
984 j = -1;
985 value = mask = nv = 0;
986 while (i < n_ev) {
987 if (j >= 0) {
988
989 value = svalues[i];
990 mask = smasks[i];
991 j = choice[i];
992 }
993
994
995
996
997 while (++j < n_alt[i]) {
998 nv = (value | cpuhw->avalues[i][j]) +
999 (value & cpuhw->avalues[i][j] & addf);
1000 if ((((nv + tadd) ^ value) & mask) == 0 &&
1001 (((nv + tadd) ^ cpuhw->avalues[i][j])
1002 & cpuhw->amasks[i][j]) == 0)
1003 break;
1004 }
1005 if (j >= n_alt[i]) {
1006
1007
1008
1009
1010
1011 if (--i < 0)
1012 return -1;
1013 } else {
1014
1015
1016
1017
1018
1019
1020 choice[i] = j;
1021 svalues[i] = value;
1022 smasks[i] = mask;
1023 value = nv;
1024 mask |= cpuhw->amasks[i][j];
1025 ++i;
1026 j = -1;
1027 }
1028 }
1029
1030
1031 for (i = 0; i < n_ev; ++i)
1032 event_id[i] = cpuhw->alternatives[i][choice[i]];
1033 return 0;
1034}
1035
1036
1037
1038
1039
1040
1041static int check_excludes(struct perf_event **ctrs, unsigned int cflags[],
1042 int n_prev, int n_new)
1043{
1044 int eu = 0, ek = 0, eh = 0;
1045 int i, n, first;
1046 struct perf_event *event;
1047
1048
1049
1050
1051
1052
1053 if (ppmu->flags & PPMU_ARCH_207S)
1054 return 0;
1055
1056 n = n_prev + n_new;
1057 if (n <= 1)
1058 return 0;
1059
1060 first = 1;
1061 for (i = 0; i < n; ++i) {
1062 if (cflags[i] & PPMU_LIMITED_PMC_OK) {
1063 cflags[i] &= ~PPMU_LIMITED_PMC_REQD;
1064 continue;
1065 }
1066 event = ctrs[i];
1067 if (first) {
1068 eu = event->attr.exclude_user;
1069 ek = event->attr.exclude_kernel;
1070 eh = event->attr.exclude_hv;
1071 first = 0;
1072 } else if (event->attr.exclude_user != eu ||
1073 event->attr.exclude_kernel != ek ||
1074 event->attr.exclude_hv != eh) {
1075 return -EAGAIN;
1076 }
1077 }
1078
1079 if (eu || ek || eh)
1080 for (i = 0; i < n; ++i)
1081 if (cflags[i] & PPMU_LIMITED_PMC_OK)
1082 cflags[i] |= PPMU_LIMITED_PMC_REQD;
1083
1084 return 0;
1085}
1086
1087static u64 check_and_compute_delta(u64 prev, u64 val)
1088{
1089 u64 delta = (val - prev) & 0xfffffffful;
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100 if (prev > val && (prev - val) < 256)
1101 delta = 0;
1102
1103 return delta;
1104}
1105
1106static void power_pmu_read(struct perf_event *event)
1107{
1108 s64 val, delta, prev;
1109
1110 if (event->hw.state & PERF_HES_STOPPED)
1111 return;
1112
1113 if (!event->hw.idx)
1114 return;
1115
1116 if (is_ebb_event(event)) {
1117 val = read_pmc(event->hw.idx);
1118 local64_set(&event->hw.prev_count, val);
1119 return;
1120 }
1121
1122
1123
1124
1125
1126
1127 do {
1128 prev = local64_read(&event->hw.prev_count);
1129 barrier();
1130 val = read_pmc(event->hw.idx);
1131 delta = check_and_compute_delta(prev, val);
1132 if (!delta)
1133 return;
1134 } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
1135
1136 local64_add(delta, &event->count);
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147 do {
1148 prev = local64_read(&event->hw.period_left);
1149 val = prev - delta;
1150 if (val < 1)
1151 val = 1;
1152 } while (local64_cmpxchg(&event->hw.period_left, prev, val) != prev);
1153}
1154
1155
1156
1157
1158
1159
1160static int is_limited_pmc(int pmcnum)
1161{
1162 return (ppmu->flags & PPMU_LIMITED_PMC5_6)
1163 && (pmcnum == 5 || pmcnum == 6);
1164}
1165
1166static void freeze_limited_counters(struct cpu_hw_events *cpuhw,
1167 unsigned long pmc5, unsigned long pmc6)
1168{
1169 struct perf_event *event;
1170 u64 val, prev, delta;
1171 int i;
1172
1173 for (i = 0; i < cpuhw->n_limited; ++i) {
1174 event = cpuhw->limited_counter[i];
1175 if (!event->hw.idx)
1176 continue;
1177 val = (event->hw.idx == 5) ? pmc5 : pmc6;
1178 prev = local64_read(&event->hw.prev_count);
1179 event->hw.idx = 0;
1180 delta = check_and_compute_delta(prev, val);
1181 if (delta)
1182 local64_add(delta, &event->count);
1183 }
1184}
1185
1186static void thaw_limited_counters(struct cpu_hw_events *cpuhw,
1187 unsigned long pmc5, unsigned long pmc6)
1188{
1189 struct perf_event *event;
1190 u64 val, prev;
1191 int i;
1192
1193 for (i = 0; i < cpuhw->n_limited; ++i) {
1194 event = cpuhw->limited_counter[i];
1195 event->hw.idx = cpuhw->limited_hwidx[i];
1196 val = (event->hw.idx == 5) ? pmc5 : pmc6;
1197 prev = local64_read(&event->hw.prev_count);
1198 if (check_and_compute_delta(prev, val))
1199 local64_set(&event->hw.prev_count, val);
1200 perf_event_update_userpage(event);
1201 }
1202}
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215static void write_mmcr0(struct cpu_hw_events *cpuhw, unsigned long mmcr0)
1216{
1217 unsigned long pmc5, pmc6;
1218
1219 if (!cpuhw->n_limited) {
1220 mtspr(SPRN_MMCR0, mmcr0);
1221 return;
1222 }
1223
1224
1225
1226
1227
1228
1229
1230
1231 asm volatile("mtspr %3,%2; mfspr %0,%4; mfspr %1,%5"
1232 : "=&r" (pmc5), "=&r" (pmc6)
1233 : "r" (mmcr0 & ~(MMCR0_PMC1CE | MMCR0_PMCjCE)),
1234 "i" (SPRN_MMCR0),
1235 "i" (SPRN_PMC5), "i" (SPRN_PMC6));
1236
1237 if (mmcr0 & MMCR0_FC)
1238 freeze_limited_counters(cpuhw, pmc5, pmc6);
1239 else
1240 thaw_limited_counters(cpuhw, pmc5, pmc6);
1241
1242
1243
1244
1245
1246 if (mmcr0 & (MMCR0_PMC1CE | MMCR0_PMCjCE))
1247 mtspr(SPRN_MMCR0, mmcr0);
1248}
1249
1250
1251
1252
1253
1254static void power_pmu_disable(struct pmu *pmu)
1255{
1256 struct cpu_hw_events *cpuhw;
1257 unsigned long flags, mmcr0, val, mmcra;
1258
1259 if (!ppmu)
1260 return;
1261 local_irq_save(flags);
1262 cpuhw = this_cpu_ptr(&cpu_hw_events);
1263
1264 if (!cpuhw->disabled) {
1265
1266
1267
1268 if (!cpuhw->pmcs_enabled) {
1269 ppc_enable_pmcs();
1270 cpuhw->pmcs_enabled = 1;
1271 }
1272
1273
1274
1275
1276 val = mmcr0 = mfspr(SPRN_MMCR0);
1277 val |= MMCR0_FC;
1278 val &= ~(MMCR0_EBE | MMCR0_BHRBA | MMCR0_PMCC | MMCR0_PMAO |
1279 MMCR0_FC56);
1280
1281 if (ppmu->flags & PPMU_ARCH_31)
1282 val |= MMCR0_PMCCEXT;
1283
1284
1285
1286
1287
1288
1289 write_mmcr0(cpuhw, val);
1290 mb();
1291 isync();
1292
1293 val = mmcra = cpuhw->mmcr.mmcra;
1294
1295
1296
1297
1298 if (cpuhw->mmcr.mmcra & MMCRA_SAMPLE_ENABLE)
1299 val &= ~MMCRA_SAMPLE_ENABLE;
1300
1301
1302 if (ppmu->flags & PPMU_ARCH_31)
1303 val |= MMCRA_BHRB_DISABLE;
1304
1305
1306
1307
1308
1309 if (val != mmcra) {
1310 mtspr(SPRN_MMCRA, mmcra);
1311 mb();
1312 isync();
1313 }
1314
1315 cpuhw->disabled = 1;
1316 cpuhw->n_added = 0;
1317
1318 ebb_switch_out(mmcr0);
1319
1320#ifdef CONFIG_PPC64
1321
1322
1323
1324
1325
1326
1327 if (ppmu->flags & PPMU_ARCH_207S) {
1328 mtspr(SPRN_SDAR, 0);
1329 mtspr(SPRN_SIAR, 0);
1330 }
1331#endif
1332 }
1333
1334 local_irq_restore(flags);
1335}
1336
1337
1338
1339
1340
1341
1342static void power_pmu_enable(struct pmu *pmu)
1343{
1344 struct perf_event *event;
1345 struct cpu_hw_events *cpuhw;
1346 unsigned long flags;
1347 long i;
1348 unsigned long val, mmcr0;
1349 s64 left;
1350 unsigned int hwc_index[MAX_HWEVENTS];
1351 int n_lim;
1352 int idx;
1353 bool ebb;
1354
1355 if (!ppmu)
1356 return;
1357 local_irq_save(flags);
1358
1359 cpuhw = this_cpu_ptr(&cpu_hw_events);
1360 if (!cpuhw->disabled)
1361 goto out;
1362
1363 if (cpuhw->n_events == 0) {
1364 ppc_set_pmu_inuse(0);
1365 goto out;
1366 }
1367
1368 cpuhw->disabled = 0;
1369
1370
1371
1372
1373
1374
1375 ebb = is_ebb_event(cpuhw->event[0]);
1376
1377
1378
1379
1380
1381
1382
1383 if (!cpuhw->n_added) {
1384 mtspr(SPRN_MMCRA, cpuhw->mmcr.mmcra & ~MMCRA_SAMPLE_ENABLE);
1385 mtspr(SPRN_MMCR1, cpuhw->mmcr.mmcr1);
1386 if (ppmu->flags & PPMU_ARCH_31)
1387 mtspr(SPRN_MMCR3, cpuhw->mmcr.mmcr3);
1388 goto out_enable;
1389 }
1390
1391
1392
1393
1394 memset(&cpuhw->mmcr, 0, sizeof(cpuhw->mmcr));
1395
1396 if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_events, hwc_index,
1397 &cpuhw->mmcr, cpuhw->event, ppmu->flags)) {
1398
1399 printk(KERN_ERR "oops compute_mmcr failed\n");
1400 goto out;
1401 }
1402
1403 if (!(ppmu->flags & PPMU_ARCH_207S)) {
1404
1405
1406
1407
1408
1409 event = cpuhw->event[0];
1410 if (event->attr.exclude_user)
1411 cpuhw->mmcr.mmcr0 |= MMCR0_FCP;
1412 if (event->attr.exclude_kernel)
1413 cpuhw->mmcr.mmcr0 |= freeze_events_kernel;
1414 if (event->attr.exclude_hv)
1415 cpuhw->mmcr.mmcr0 |= MMCR0_FCHV;
1416 }
1417
1418
1419
1420
1421
1422
1423 ppc_set_pmu_inuse(1);
1424 mtspr(SPRN_MMCRA, cpuhw->mmcr.mmcra & ~MMCRA_SAMPLE_ENABLE);
1425 mtspr(SPRN_MMCR1, cpuhw->mmcr.mmcr1);
1426 mtspr(SPRN_MMCR0, (cpuhw->mmcr.mmcr0 & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
1427 | MMCR0_FC);
1428 if (ppmu->flags & PPMU_ARCH_207S)
1429 mtspr(SPRN_MMCR2, cpuhw->mmcr.mmcr2);
1430
1431 if (ppmu->flags & PPMU_ARCH_31)
1432 mtspr(SPRN_MMCR3, cpuhw->mmcr.mmcr3);
1433
1434
1435
1436
1437
1438 for (i = 0; i < cpuhw->n_events; ++i) {
1439 event = cpuhw->event[i];
1440 if (event->hw.idx && event->hw.idx != hwc_index[i] + 1) {
1441 power_pmu_read(event);
1442 write_pmc(event->hw.idx, 0);
1443 event->hw.idx = 0;
1444 }
1445 }
1446
1447
1448
1449
1450 cpuhw->n_limited = n_lim = 0;
1451 for (i = 0; i < cpuhw->n_events; ++i) {
1452 event = cpuhw->event[i];
1453 if (event->hw.idx)
1454 continue;
1455 idx = hwc_index[i] + 1;
1456 if (is_limited_pmc(idx)) {
1457 cpuhw->limited_counter[n_lim] = event;
1458 cpuhw->limited_hwidx[n_lim] = idx;
1459 ++n_lim;
1460 continue;
1461 }
1462
1463 if (ebb)
1464 val = local64_read(&event->hw.prev_count);
1465 else {
1466 val = 0;
1467 if (event->hw.sample_period) {
1468 left = local64_read(&event->hw.period_left);
1469 if (left < 0x80000000L)
1470 val = 0x80000000L - left;
1471 }
1472 local64_set(&event->hw.prev_count, val);
1473 }
1474
1475 event->hw.idx = idx;
1476 if (event->hw.state & PERF_HES_STOPPED)
1477 val = 0;
1478 write_pmc(idx, val);
1479
1480 perf_event_update_userpage(event);
1481 }
1482 cpuhw->n_limited = n_lim;
1483 cpuhw->mmcr.mmcr0 |= MMCR0_PMXE | MMCR0_FCECE;
1484
1485 out_enable:
1486 pmao_restore_workaround(ebb);
1487
1488 mmcr0 = ebb_switch_in(ebb, cpuhw);
1489
1490 mb();
1491 if (cpuhw->bhrb_users)
1492 ppmu->config_bhrb(cpuhw->bhrb_filter);
1493
1494 write_mmcr0(cpuhw, mmcr0);
1495
1496
1497
1498
1499 if (cpuhw->mmcr.mmcra & MMCRA_SAMPLE_ENABLE) {
1500 mb();
1501 mtspr(SPRN_MMCRA, cpuhw->mmcr.mmcra);
1502 }
1503
1504 out:
1505
1506 local_irq_restore(flags);
1507}
1508
1509static int collect_events(struct perf_event *group, int max_count,
1510 struct perf_event *ctrs[], u64 *events,
1511 unsigned int *flags)
1512{
1513 int n = 0;
1514 struct perf_event *event;
1515
1516 if (group->pmu->task_ctx_nr == perf_hw_context) {
1517 if (n >= max_count)
1518 return -1;
1519 ctrs[n] = group;
1520 flags[n] = group->hw.event_base;
1521 events[n++] = group->hw.config;
1522 }
1523 for_each_sibling_event(event, group) {
1524 if (event->pmu->task_ctx_nr == perf_hw_context &&
1525 event->state != PERF_EVENT_STATE_OFF) {
1526 if (n >= max_count)
1527 return -1;
1528 ctrs[n] = event;
1529 flags[n] = event->hw.event_base;
1530 events[n++] = event->hw.config;
1531 }
1532 }
1533 return n;
1534}
1535
1536
1537
1538
1539
1540
1541
1542static int power_pmu_add(struct perf_event *event, int ef_flags)
1543{
1544 struct cpu_hw_events *cpuhw;
1545 unsigned long flags;
1546 int n0;
1547 int ret = -EAGAIN;
1548
1549 local_irq_save(flags);
1550 perf_pmu_disable(event->pmu);
1551
1552
1553
1554
1555
1556 cpuhw = this_cpu_ptr(&cpu_hw_events);
1557 n0 = cpuhw->n_events;
1558 if (n0 >= ppmu->n_counter)
1559 goto out;
1560 cpuhw->event[n0] = event;
1561 cpuhw->events[n0] = event->hw.config;
1562 cpuhw->flags[n0] = event->hw.event_base;
1563
1564
1565
1566
1567
1568
1569
1570 if (!(ef_flags & PERF_EF_START))
1571 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
1572 else
1573 event->hw.state = 0;
1574
1575
1576
1577
1578
1579
1580 if (cpuhw->txn_flags & PERF_PMU_TXN_ADD)
1581 goto nocheck;
1582
1583 if (check_excludes(cpuhw->event, cpuhw->flags, n0, 1))
1584 goto out;
1585 if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1, cpuhw->event))
1586 goto out;
1587 event->hw.config = cpuhw->events[n0];
1588
1589nocheck:
1590 ebb_event_add(event);
1591
1592 ++cpuhw->n_events;
1593 ++cpuhw->n_added;
1594
1595 ret = 0;
1596 out:
1597 if (has_branch_stack(event)) {
1598 u64 bhrb_filter = -1;
1599
1600 if (ppmu->bhrb_filter_map)
1601 bhrb_filter = ppmu->bhrb_filter_map(
1602 event->attr.branch_sample_type);
1603
1604 if (bhrb_filter != -1) {
1605 cpuhw->bhrb_filter = bhrb_filter;
1606 power_pmu_bhrb_enable(event);
1607 }
1608 }
1609
1610 perf_pmu_enable(event->pmu);
1611 local_irq_restore(flags);
1612 return ret;
1613}
1614
1615
1616
1617
1618static void power_pmu_del(struct perf_event *event, int ef_flags)
1619{
1620 struct cpu_hw_events *cpuhw;
1621 long i;
1622 unsigned long flags;
1623
1624 local_irq_save(flags);
1625 perf_pmu_disable(event->pmu);
1626
1627 power_pmu_read(event);
1628
1629 cpuhw = this_cpu_ptr(&cpu_hw_events);
1630 for (i = 0; i < cpuhw->n_events; ++i) {
1631 if (event == cpuhw->event[i]) {
1632 while (++i < cpuhw->n_events) {
1633 cpuhw->event[i-1] = cpuhw->event[i];
1634 cpuhw->events[i-1] = cpuhw->events[i];
1635 cpuhw->flags[i-1] = cpuhw->flags[i];
1636 }
1637 --cpuhw->n_events;
1638 ppmu->disable_pmc(event->hw.idx - 1, &cpuhw->mmcr);
1639 if (event->hw.idx) {
1640 write_pmc(event->hw.idx, 0);
1641 event->hw.idx = 0;
1642 }
1643 perf_event_update_userpage(event);
1644 break;
1645 }
1646 }
1647 for (i = 0; i < cpuhw->n_limited; ++i)
1648 if (event == cpuhw->limited_counter[i])
1649 break;
1650 if (i < cpuhw->n_limited) {
1651 while (++i < cpuhw->n_limited) {
1652 cpuhw->limited_counter[i-1] = cpuhw->limited_counter[i];
1653 cpuhw->limited_hwidx[i-1] = cpuhw->limited_hwidx[i];
1654 }
1655 --cpuhw->n_limited;
1656 }
1657 if (cpuhw->n_events == 0) {
1658
1659 cpuhw->mmcr.mmcr0 &= ~(MMCR0_PMXE | MMCR0_FCECE);
1660 }
1661
1662 if (has_branch_stack(event))
1663 power_pmu_bhrb_disable(event);
1664
1665 perf_pmu_enable(event->pmu);
1666 local_irq_restore(flags);
1667}
1668
1669
1670
1671
1672
1673
1674static void power_pmu_start(struct perf_event *event, int ef_flags)
1675{
1676 unsigned long flags;
1677 s64 left;
1678 unsigned long val;
1679
1680 if (!event->hw.idx || !event->hw.sample_period)
1681 return;
1682
1683 if (!(event->hw.state & PERF_HES_STOPPED))
1684 return;
1685
1686 if (ef_flags & PERF_EF_RELOAD)
1687 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1688
1689 local_irq_save(flags);
1690 perf_pmu_disable(event->pmu);
1691
1692 event->hw.state = 0;
1693 left = local64_read(&event->hw.period_left);
1694
1695 val = 0;
1696 if (left < 0x80000000L)
1697 val = 0x80000000L - left;
1698
1699 write_pmc(event->hw.idx, val);
1700
1701 perf_event_update_userpage(event);
1702 perf_pmu_enable(event->pmu);
1703 local_irq_restore(flags);
1704}
1705
1706static void power_pmu_stop(struct perf_event *event, int ef_flags)
1707{
1708 unsigned long flags;
1709
1710 if (!event->hw.idx || !event->hw.sample_period)
1711 return;
1712
1713 if (event->hw.state & PERF_HES_STOPPED)
1714 return;
1715
1716 local_irq_save(flags);
1717 perf_pmu_disable(event->pmu);
1718
1719 power_pmu_read(event);
1720 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
1721 write_pmc(event->hw.idx, 0);
1722
1723 perf_event_update_userpage(event);
1724 perf_pmu_enable(event->pmu);
1725 local_irq_restore(flags);
1726}
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737static void power_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
1738{
1739 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
1740
1741 WARN_ON_ONCE(cpuhw->txn_flags);
1742
1743 cpuhw->txn_flags = txn_flags;
1744 if (txn_flags & ~PERF_PMU_TXN_ADD)
1745 return;
1746
1747 perf_pmu_disable(pmu);
1748 cpuhw->n_txn_start = cpuhw->n_events;
1749}
1750
1751
1752
1753
1754
1755
1756static void power_pmu_cancel_txn(struct pmu *pmu)
1757{
1758 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
1759 unsigned int txn_flags;
1760
1761 WARN_ON_ONCE(!cpuhw->txn_flags);
1762
1763 txn_flags = cpuhw->txn_flags;
1764 cpuhw->txn_flags = 0;
1765 if (txn_flags & ~PERF_PMU_TXN_ADD)
1766 return;
1767
1768 perf_pmu_enable(pmu);
1769}
1770
1771
1772
1773
1774
1775
1776static int power_pmu_commit_txn(struct pmu *pmu)
1777{
1778 struct cpu_hw_events *cpuhw;
1779 long i, n;
1780
1781 if (!ppmu)
1782 return -EAGAIN;
1783
1784 cpuhw = this_cpu_ptr(&cpu_hw_events);
1785 WARN_ON_ONCE(!cpuhw->txn_flags);
1786
1787 if (cpuhw->txn_flags & ~PERF_PMU_TXN_ADD) {
1788 cpuhw->txn_flags = 0;
1789 return 0;
1790 }
1791
1792 n = cpuhw->n_events;
1793 if (check_excludes(cpuhw->event, cpuhw->flags, 0, n))
1794 return -EAGAIN;
1795 i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n, cpuhw->event);
1796 if (i < 0)
1797 return -EAGAIN;
1798
1799 for (i = cpuhw->n_txn_start; i < n; ++i)
1800 cpuhw->event[i]->hw.config = cpuhw->events[i];
1801
1802 cpuhw->txn_flags = 0;
1803 perf_pmu_enable(pmu);
1804 return 0;
1805}
1806
1807
1808
1809
1810
1811
1812
1813
1814static int can_go_on_limited_pmc(struct perf_event *event, u64 ev,
1815 unsigned int flags)
1816{
1817 int n;
1818 u64 alt[MAX_EVENT_ALTERNATIVES];
1819
1820 if (event->attr.exclude_user
1821 || event->attr.exclude_kernel
1822 || event->attr.exclude_hv
1823 || event->attr.sample_period)
1824 return 0;
1825
1826 if (ppmu->limited_pmc_event(ev))
1827 return 1;
1828
1829
1830
1831
1832
1833 if (!ppmu->get_alternatives)
1834 return 0;
1835
1836 flags |= PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD;
1837 n = ppmu->get_alternatives(ev, flags, alt);
1838
1839 return n > 0;
1840}
1841
1842
1843
1844
1845
1846
1847static u64 normal_pmc_alternative(u64 ev, unsigned long flags)
1848{
1849 u64 alt[MAX_EVENT_ALTERNATIVES];
1850 int n;
1851
1852 flags &= ~(PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD);
1853 n = ppmu->get_alternatives(ev, flags, alt);
1854 if (!n)
1855 return 0;
1856 return alt[0];
1857}
1858
1859
1860static atomic_t num_events;
1861
1862static DEFINE_MUTEX(pmc_reserve_mutex);
1863
1864
1865
1866
1867static void hw_perf_event_destroy(struct perf_event *event)
1868{
1869 if (!atomic_add_unless(&num_events, -1, 1)) {
1870 mutex_lock(&pmc_reserve_mutex);
1871 if (atomic_dec_return(&num_events) == 0)
1872 release_pmc_hardware();
1873 mutex_unlock(&pmc_reserve_mutex);
1874 }
1875}
1876
1877
1878
1879
1880static int hw_perf_cache_event(u64 config, u64 *eventp)
1881{
1882 unsigned long type, op, result;
1883 u64 ev;
1884
1885 if (!ppmu->cache_events)
1886 return -EINVAL;
1887
1888
1889 type = config & 0xff;
1890 op = (config >> 8) & 0xff;
1891 result = (config >> 16) & 0xff;
1892
1893 if (type >= PERF_COUNT_HW_CACHE_MAX ||
1894 op >= PERF_COUNT_HW_CACHE_OP_MAX ||
1895 result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
1896 return -EINVAL;
1897
1898 ev = (*ppmu->cache_events)[type][op][result];
1899 if (ev == 0)
1900 return -EOPNOTSUPP;
1901 if (ev == -1)
1902 return -EINVAL;
1903 *eventp = ev;
1904 return 0;
1905}
1906
1907static bool is_event_blacklisted(u64 ev)
1908{
1909 int i;
1910
1911 for (i=0; i < ppmu->n_blacklist_ev; i++) {
1912 if (ppmu->blacklist_ev[i] == ev)
1913 return true;
1914 }
1915
1916 return false;
1917}
1918
1919static int power_pmu_event_init(struct perf_event *event)
1920{
1921 u64 ev;
1922 unsigned long flags, irq_flags;
1923 struct perf_event *ctrs[MAX_HWEVENTS];
1924 u64 events[MAX_HWEVENTS];
1925 unsigned int cflags[MAX_HWEVENTS];
1926 int n;
1927 int err;
1928 struct cpu_hw_events *cpuhw;
1929
1930 if (!ppmu)
1931 return -ENOENT;
1932
1933 if (has_branch_stack(event)) {
1934
1935 if (!(ppmu->flags & PPMU_ARCH_207S))
1936 return -EOPNOTSUPP;
1937 }
1938
1939 switch (event->attr.type) {
1940 case PERF_TYPE_HARDWARE:
1941 ev = event->attr.config;
1942 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
1943 return -EOPNOTSUPP;
1944
1945 if (ppmu->blacklist_ev && is_event_blacklisted(ev))
1946 return -EINVAL;
1947 ev = ppmu->generic_events[ev];
1948 break;
1949 case PERF_TYPE_HW_CACHE:
1950 err = hw_perf_cache_event(event->attr.config, &ev);
1951 if (err)
1952 return err;
1953
1954 if (ppmu->blacklist_ev && is_event_blacklisted(ev))
1955 return -EINVAL;
1956 break;
1957 case PERF_TYPE_RAW:
1958 ev = event->attr.config;
1959
1960 if (ppmu->blacklist_ev && is_event_blacklisted(ev))
1961 return -EINVAL;
1962 break;
1963 default:
1964 return -ENOENT;
1965 }
1966
1967
1968
1969
1970
1971
1972
1973
1974 if (ppmu->check_attr_config &&
1975 ppmu->check_attr_config(event))
1976 return -EINVAL;
1977
1978 event->hw.config_base = ev;
1979 event->hw.idx = 0;
1980
1981
1982
1983
1984
1985
1986 if (!firmware_has_feature(FW_FEATURE_LPAR))
1987 event->attr.exclude_hv = 0;
1988
1989
1990
1991
1992
1993
1994
1995 flags = 0;
1996 if (event->attach_state & PERF_ATTACH_TASK)
1997 flags |= PPMU_ONLY_COUNT_RUN;
1998
1999
2000
2001
2002
2003 if (ppmu->flags & PPMU_LIMITED_PMC5_6) {
2004 if (can_go_on_limited_pmc(event, ev, flags)) {
2005 flags |= PPMU_LIMITED_PMC_OK;
2006 } else if (ppmu->limited_pmc_event(ev)) {
2007
2008
2009
2010
2011
2012 ev = normal_pmc_alternative(ev, flags);
2013 if (!ev)
2014 return -EINVAL;
2015 }
2016 }
2017
2018
2019 err = ebb_event_check(event);
2020 if (err)
2021 return err;
2022
2023
2024
2025
2026
2027
2028 n = 0;
2029 if (event->group_leader != event) {
2030 n = collect_events(event->group_leader, ppmu->n_counter - 1,
2031 ctrs, events, cflags);
2032 if (n < 0)
2033 return -EINVAL;
2034 }
2035 events[n] = ev;
2036 ctrs[n] = event;
2037 cflags[n] = flags;
2038 if (check_excludes(ctrs, cflags, n, 1))
2039 return -EINVAL;
2040
2041 local_irq_save(irq_flags);
2042 cpuhw = this_cpu_ptr(&cpu_hw_events);
2043
2044 err = power_check_constraints(cpuhw, events, cflags, n + 1, ctrs);
2045
2046 if (has_branch_stack(event)) {
2047 u64 bhrb_filter = -1;
2048
2049 if (ppmu->bhrb_filter_map)
2050 bhrb_filter = ppmu->bhrb_filter_map(
2051 event->attr.branch_sample_type);
2052
2053 if (bhrb_filter == -1) {
2054 local_irq_restore(irq_flags);
2055 return -EOPNOTSUPP;
2056 }
2057 cpuhw->bhrb_filter = bhrb_filter;
2058 }
2059
2060 local_irq_restore(irq_flags);
2061 if (err)
2062 return -EINVAL;
2063
2064 event->hw.config = events[n];
2065 event->hw.event_base = cflags[n];
2066 event->hw.last_period = event->hw.sample_period;
2067 local64_set(&event->hw.period_left, event->hw.last_period);
2068
2069
2070
2071
2072
2073 if (is_ebb_event(event))
2074 local64_set(&event->hw.prev_count, 0);
2075
2076
2077
2078
2079
2080
2081
2082 err = 0;
2083 if (!atomic_inc_not_zero(&num_events)) {
2084 mutex_lock(&pmc_reserve_mutex);
2085 if (atomic_read(&num_events) == 0 &&
2086 reserve_pmc_hardware(perf_event_interrupt))
2087 err = -EBUSY;
2088 else
2089 atomic_inc(&num_events);
2090 mutex_unlock(&pmc_reserve_mutex);
2091 }
2092 event->destroy = hw_perf_event_destroy;
2093
2094 return err;
2095}
2096
2097static int power_pmu_event_idx(struct perf_event *event)
2098{
2099 return event->hw.idx;
2100}
2101
2102ssize_t power_events_sysfs_show(struct device *dev,
2103 struct device_attribute *attr, char *page)
2104{
2105 struct perf_pmu_events_attr *pmu_attr;
2106
2107 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
2108
2109 return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
2110}
2111
2112static struct pmu power_pmu = {
2113 .pmu_enable = power_pmu_enable,
2114 .pmu_disable = power_pmu_disable,
2115 .event_init = power_pmu_event_init,
2116 .add = power_pmu_add,
2117 .del = power_pmu_del,
2118 .start = power_pmu_start,
2119 .stop = power_pmu_stop,
2120 .read = power_pmu_read,
2121 .start_txn = power_pmu_start_txn,
2122 .cancel_txn = power_pmu_cancel_txn,
2123 .commit_txn = power_pmu_commit_txn,
2124 .event_idx = power_pmu_event_idx,
2125 .sched_task = power_pmu_sched_task,
2126};
2127
2128#define PERF_SAMPLE_ADDR_TYPE (PERF_SAMPLE_ADDR | \
2129 PERF_SAMPLE_PHYS_ADDR | \
2130 PERF_SAMPLE_DATA_PAGE_SIZE)
2131
2132
2133
2134
2135
2136static void record_and_restart(struct perf_event *event, unsigned long val,
2137 struct pt_regs *regs)
2138{
2139 u64 period = event->hw.sample_period;
2140 s64 prev, delta, left;
2141 int record = 0;
2142
2143 if (event->hw.state & PERF_HES_STOPPED) {
2144 write_pmc(event->hw.idx, 0);
2145 return;
2146 }
2147
2148
2149 prev = local64_read(&event->hw.prev_count);
2150 delta = check_and_compute_delta(prev, val);
2151 local64_add(delta, &event->count);
2152
2153
2154
2155
2156
2157 val = 0;
2158 left = local64_read(&event->hw.period_left) - delta;
2159 if (delta == 0)
2160 left++;
2161 if (period) {
2162 if (left <= 0) {
2163 left += period;
2164 if (left <= 0)
2165 left = period;
2166
2167
2168
2169
2170
2171
2172 if (event->attr.sample_type & PERF_SAMPLE_IP)
2173 record = siar_valid(regs);
2174 else
2175 record = 1;
2176
2177 event->hw.last_period = event->hw.sample_period;
2178 }
2179 if (left < 0x80000000LL)
2180 val = 0x80000000LL - left;
2181 }
2182
2183 write_pmc(event->hw.idx, val);
2184 local64_set(&event->hw.prev_count, val);
2185 local64_set(&event->hw.period_left, left);
2186 perf_event_update_userpage(event);
2187
2188
2189
2190
2191
2192
2193
2194 if (event->attr.exclude_kernel &&
2195 (event->attr.sample_type & PERF_SAMPLE_IP) &&
2196 is_kernel_addr(mfspr(SPRN_SIAR)))
2197 record = 0;
2198
2199
2200
2201
2202 if (record) {
2203 struct perf_sample_data data;
2204
2205 perf_sample_data_init(&data, ~0ULL, event->hw.last_period);
2206
2207 if (event->attr.sample_type & PERF_SAMPLE_ADDR_TYPE)
2208 perf_get_data_addr(event, regs, &data.addr);
2209
2210 if (event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK) {
2211 struct cpu_hw_events *cpuhw;
2212 cpuhw = this_cpu_ptr(&cpu_hw_events);
2213 power_pmu_bhrb_read(event, cpuhw);
2214 data.br_stack = &cpuhw->bhrb_stack;
2215 }
2216
2217 if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC &&
2218 ppmu->get_mem_data_src)
2219 ppmu->get_mem_data_src(&data.data_src, ppmu->flags, regs);
2220
2221 if (event->attr.sample_type & PERF_SAMPLE_WEIGHT_TYPE &&
2222 ppmu->get_mem_weight)
2223 ppmu->get_mem_weight(&data.weight.full, event->attr.sample_type);
2224
2225 if (perf_event_overflow(event, &data, regs))
2226 power_pmu_stop(event, 0);
2227 } else if (period) {
2228
2229 if (perf_event_account_interrupt(event))
2230 power_pmu_stop(event, 0);
2231 }
2232}
2233
2234
2235
2236
2237
2238unsigned long perf_misc_flags(struct pt_regs *regs)
2239{
2240 u32 flags = perf_get_misc_flags(regs);
2241
2242 if (flags)
2243 return flags;
2244 return user_mode(regs) ? PERF_RECORD_MISC_USER :
2245 PERF_RECORD_MISC_KERNEL;
2246}
2247
2248
2249
2250
2251
2252unsigned long perf_instruction_pointer(struct pt_regs *regs)
2253{
2254 bool use_siar = regs_use_siar(regs);
2255 unsigned long siar = mfspr(SPRN_SIAR);
2256
2257 if (ppmu && (ppmu->flags & PPMU_P10_DD1)) {
2258 if (siar)
2259 return siar;
2260 else
2261 return regs->nip;
2262 } else if (use_siar && siar_valid(regs))
2263 return mfspr(SPRN_SIAR) + perf_ip_adjust(regs);
2264 else if (use_siar)
2265 return 0;
2266 else
2267 return regs->nip;
2268}
2269
2270static bool pmc_overflow_power7(unsigned long val)
2271{
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283 if ((0x80000000 - val) <= 256)
2284 return true;
2285
2286 return false;
2287}
2288
2289static bool pmc_overflow(unsigned long val)
2290{
2291 if ((int)val < 0)
2292 return true;
2293
2294 return false;
2295}
2296
2297
2298
2299
2300static void __perf_event_interrupt(struct pt_regs *regs)
2301{
2302 int i, j;
2303 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
2304 struct perf_event *event;
2305 int found, active;
2306
2307 if (cpuhw->n_limited)
2308 freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5),
2309 mfspr(SPRN_PMC6));
2310
2311 perf_read_regs(regs);
2312
2313
2314 for (i = 0; i < ppmu->n_counter; ++i)
2315 cpuhw->pmcs[i] = read_pmc(i + 1);
2316
2317
2318 found = 0;
2319 for (i = 0; i < ppmu->n_counter; ++i) {
2320 if (!pmc_overflow(cpuhw->pmcs[i]))
2321 continue;
2322 if (is_limited_pmc(i + 1))
2323 continue;
2324
2325
2326
2327
2328
2329 found = 1;
2330 active = 0;
2331 for (j = 0; j < cpuhw->n_events; ++j) {
2332 event = cpuhw->event[j];
2333 if (event->hw.idx == (i + 1)) {
2334 active = 1;
2335 record_and_restart(event, cpuhw->pmcs[i], regs);
2336 break;
2337 }
2338 }
2339 if (!active)
2340
2341 write_pmc(i + 1, 0);
2342 }
2343 if (!found && pvr_version_is(PVR_POWER7)) {
2344
2345 for (i = 0; i < cpuhw->n_events; ++i) {
2346 event = cpuhw->event[i];
2347 if (!event->hw.idx || is_limited_pmc(event->hw.idx))
2348 continue;
2349 if (pmc_overflow_power7(cpuhw->pmcs[event->hw.idx - 1])) {
2350
2351 found = 1;
2352 record_and_restart(event,
2353 cpuhw->pmcs[event->hw.idx - 1],
2354 regs);
2355 }
2356 }
2357 }
2358 if (unlikely(!found) && !arch_irq_disabled_regs(regs))
2359 printk_ratelimited(KERN_WARNING "Can't find PMC that caused IRQ\n");
2360
2361
2362
2363
2364
2365
2366
2367
2368 write_mmcr0(cpuhw, cpuhw->mmcr.mmcr0);
2369
2370
2371 memset(&cpuhw->pmcs, 0, sizeof(cpuhw->pmcs));
2372
2373}
2374
2375static void perf_event_interrupt(struct pt_regs *regs)
2376{
2377 u64 start_clock = sched_clock();
2378
2379 __perf_event_interrupt(regs);
2380 perf_sample_event_took(sched_clock() - start_clock);
2381}
2382
2383static int power_pmu_prepare_cpu(unsigned int cpu)
2384{
2385 struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
2386
2387 if (ppmu) {
2388 memset(cpuhw, 0, sizeof(*cpuhw));
2389 cpuhw->mmcr.mmcr0 = MMCR0_FC;
2390 }
2391 return 0;
2392}
2393
2394int register_power_pmu(struct power_pmu *pmu)
2395{
2396 if (ppmu)
2397 return -EBUSY;
2398
2399 ppmu = pmu;
2400 pr_info("%s performance monitor hardware support registered\n",
2401 pmu->name);
2402
2403 power_pmu.attr_groups = ppmu->attr_groups;
2404 power_pmu.capabilities |= (ppmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS);
2405
2406#ifdef MSR_HV
2407
2408
2409
2410 if (mfmsr() & MSR_HV)
2411 freeze_events_kernel = MMCR0_FCHV;
2412#endif
2413
2414 perf_pmu_register(&power_pmu, "cpu", PERF_TYPE_RAW);
2415 cpuhp_setup_state(CPUHP_PERF_POWER, "perf/powerpc:prepare",
2416 power_pmu_prepare_cpu, NULL);
2417 return 0;
2418}
2419
2420#ifdef CONFIG_PPC64
2421static int __init init_ppc64_pmu(void)
2422{
2423
2424 if (!init_power5_pmu())
2425 return 0;
2426 else if (!init_power5p_pmu())
2427 return 0;
2428 else if (!init_power6_pmu())
2429 return 0;
2430 else if (!init_power7_pmu())
2431 return 0;
2432 else if (!init_power8_pmu())
2433 return 0;
2434 else if (!init_power9_pmu())
2435 return 0;
2436 else if (!init_power10_pmu())
2437 return 0;
2438 else if (!init_ppc970_pmu())
2439 return 0;
2440 else
2441 return init_generic_compat_pmu();
2442}
2443early_initcall(init_ppc64_pmu);
2444#endif
2445