1
2
3
4
5
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include "uncore.h"
9#include "uncore_discovery.h"
10
11static struct rb_root discovery_tables = RB_ROOT;
12static int num_discovered_types[UNCORE_ACCESS_MAX];
13
14static bool has_generic_discovery_table(void)
15{
16 struct pci_dev *dev;
17 int dvsec;
18
19 dev = pci_get_device(PCI_VENDOR_ID_INTEL, UNCORE_DISCOVERY_TABLE_DEVICE, NULL);
20 if (!dev)
21 return false;
22
23
24 dvsec = pci_find_next_ext_capability(dev, 0, UNCORE_EXT_CAP_ID_DISCOVERY);
25 pci_dev_put(dev);
26 if (dvsec)
27 return true;
28
29 return false;
30}
31
32static int logical_die_id;
33
34static int get_device_die_id(struct pci_dev *dev)
35{
36 int cpu, node = pcibus_to_node(dev->bus);
37
38
39
40
41
42
43 if (node < 0)
44 return logical_die_id++;
45
46 for_each_cpu(cpu, cpumask_of_node(node)) {
47 struct cpuinfo_x86 *c = &cpu_data(cpu);
48
49 if (c->initialized && cpu_to_node(cpu) == node)
50 return c->logical_die_id;
51 }
52
53
54
55
56
57
58 return -1;
59}
60
61#define __node_2_type(cur) \
62 rb_entry((cur), struct intel_uncore_discovery_type, node)
63
64static inline int __type_cmp(const void *key, const struct rb_node *b)
65{
66 struct intel_uncore_discovery_type *type_b = __node_2_type(b);
67 const u16 *type_id = key;
68
69 if (type_b->type > *type_id)
70 return -1;
71 else if (type_b->type < *type_id)
72 return 1;
73
74 return 0;
75}
76
77static inline struct intel_uncore_discovery_type *
78search_uncore_discovery_type(u16 type_id)
79{
80 struct rb_node *node = rb_find(&type_id, &discovery_tables, __type_cmp);
81
82 return (node) ? __node_2_type(node) : NULL;
83}
84
85static inline bool __type_less(struct rb_node *a, const struct rb_node *b)
86{
87 return (__node_2_type(a)->type < __node_2_type(b)->type);
88}
89
90static struct intel_uncore_discovery_type *
91add_uncore_discovery_type(struct uncore_unit_discovery *unit)
92{
93 struct intel_uncore_discovery_type *type;
94
95 if (unit->access_type >= UNCORE_ACCESS_MAX) {
96 pr_warn("Unsupported access type %d\n", unit->access_type);
97 return NULL;
98 }
99
100 type = kzalloc(sizeof(struct intel_uncore_discovery_type), GFP_KERNEL);
101 if (!type)
102 return NULL;
103
104 type->box_ctrl_die = kcalloc(__uncore_max_dies, sizeof(u64), GFP_KERNEL);
105 if (!type->box_ctrl_die)
106 goto free_type;
107
108 type->access_type = unit->access_type;
109 num_discovered_types[type->access_type]++;
110 type->type = unit->box_type;
111
112 rb_add(&type->node, &discovery_tables, __type_less);
113
114 return type;
115
116free_type:
117 kfree(type);
118
119 return NULL;
120
121}
122
123static struct intel_uncore_discovery_type *
124get_uncore_discovery_type(struct uncore_unit_discovery *unit)
125{
126 struct intel_uncore_discovery_type *type;
127
128 type = search_uncore_discovery_type(unit->box_type);
129 if (type)
130 return type;
131
132 return add_uncore_discovery_type(unit);
133}
134
135static void
136uncore_insert_box_info(struct uncore_unit_discovery *unit,
137 int die, bool parsed)
138{
139 struct intel_uncore_discovery_type *type;
140 unsigned int *box_offset, *ids;
141 int i;
142
143 if (WARN_ON_ONCE(!unit->ctl || !unit->ctl_offset || !unit->ctr_offset))
144 return;
145
146 if (parsed) {
147 type = search_uncore_discovery_type(unit->box_type);
148 if (WARN_ON_ONCE(!type))
149 return;
150
151 if (!type->box_ctrl_die[die])
152 type->box_ctrl_die[die] = unit->ctl;
153 return;
154 }
155
156 type = get_uncore_discovery_type(unit);
157 if (!type)
158 return;
159
160 box_offset = kcalloc(type->num_boxes + 1, sizeof(unsigned int), GFP_KERNEL);
161 if (!box_offset)
162 return;
163
164 ids = kcalloc(type->num_boxes + 1, sizeof(unsigned int), GFP_KERNEL);
165 if (!ids)
166 goto free_box_offset;
167
168
169 if (!type->num_boxes) {
170 type->box_ctrl = unit->ctl;
171 type->box_ctrl_die[die] = unit->ctl;
172 type->num_counters = unit->num_regs;
173 type->counter_width = unit->bit_width;
174 type->ctl_offset = unit->ctl_offset;
175 type->ctr_offset = unit->ctr_offset;
176 *ids = unit->box_id;
177 goto end;
178 }
179
180 for (i = 0; i < type->num_boxes; i++) {
181 ids[i] = type->ids[i];
182 box_offset[i] = type->box_offset[i];
183
184 if (WARN_ON_ONCE(unit->box_id == ids[i]))
185 goto free_ids;
186 }
187 ids[i] = unit->box_id;
188 box_offset[i] = unit->ctl - type->box_ctrl;
189 kfree(type->ids);
190 kfree(type->box_offset);
191end:
192 type->ids = ids;
193 type->box_offset = box_offset;
194 type->num_boxes++;
195 return;
196
197free_ids:
198 kfree(ids);
199
200free_box_offset:
201 kfree(box_offset);
202
203}
204
205static int parse_discovery_table(struct pci_dev *dev, int die,
206 u32 bar_offset, bool *parsed)
207{
208 struct uncore_global_discovery global;
209 struct uncore_unit_discovery unit;
210 void __iomem *io_addr;
211 resource_size_t addr;
212 unsigned long size;
213 u32 val;
214 int i;
215
216 pci_read_config_dword(dev, bar_offset, &val);
217
218 if (val & UNCORE_DISCOVERY_MASK)
219 return -EINVAL;
220
221 addr = (resource_size_t)(val & ~UNCORE_DISCOVERY_MASK);
222 size = UNCORE_DISCOVERY_GLOBAL_MAP_SIZE;
223 io_addr = ioremap(addr, size);
224 if (!io_addr)
225 return -ENOMEM;
226
227
228 memcpy_fromio(&global, io_addr, sizeof(struct uncore_global_discovery));
229 if (uncore_discovery_invalid_unit(global)) {
230 pr_info("Invalid Global Discovery State: 0x%llx 0x%llx 0x%llx\n",
231 global.table1, global.ctl, global.table3);
232 iounmap(io_addr);
233 return -EINVAL;
234 }
235 iounmap(io_addr);
236
237 size = (1 + global.max_units) * global.stride * 8;
238 io_addr = ioremap(addr, size);
239 if (!io_addr)
240 return -ENOMEM;
241
242
243 for (i = 0; i < global.max_units; i++) {
244 memcpy_fromio(&unit, io_addr + (i + 1) * (global.stride * 8),
245 sizeof(struct uncore_unit_discovery));
246
247 if (uncore_discovery_invalid_unit(unit))
248 continue;
249
250 if (unit.access_type >= UNCORE_ACCESS_MAX)
251 continue;
252
253 uncore_insert_box_info(&unit, die, *parsed);
254 }
255
256 *parsed = true;
257 iounmap(io_addr);
258 return 0;
259}
260
261bool intel_uncore_has_discovery_tables(void)
262{
263 u32 device, val, entry_id, bar_offset;
264 int die, dvsec = 0, ret = true;
265 struct pci_dev *dev = NULL;
266 bool parsed = false;
267
268 if (has_generic_discovery_table())
269 device = UNCORE_DISCOVERY_TABLE_DEVICE;
270 else
271 device = PCI_ANY_ID;
272
273
274
275
276
277 while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, dev)) != NULL) {
278 while ((dvsec = pci_find_next_ext_capability(dev, dvsec, UNCORE_EXT_CAP_ID_DISCOVERY))) {
279 pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC_OFFSET, &val);
280 entry_id = val & UNCORE_DISCOVERY_DVSEC_ID_MASK;
281 if (entry_id != UNCORE_DISCOVERY_DVSEC_ID_PMON)
282 continue;
283
284 pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC2_OFFSET, &val);
285
286 if (val & ~UNCORE_DISCOVERY_DVSEC2_BIR_MASK) {
287 ret = false;
288 goto err;
289 }
290 bar_offset = UNCORE_DISCOVERY_BIR_BASE +
291 (val & UNCORE_DISCOVERY_DVSEC2_BIR_MASK) * UNCORE_DISCOVERY_BIR_STEP;
292
293 die = get_device_die_id(dev);
294 if (die < 0)
295 continue;
296
297 parse_discovery_table(dev, die, bar_offset, &parsed);
298 }
299 }
300
301
302 if (!parsed)
303 ret = false;
304err:
305 pci_dev_put(dev);
306
307 return ret;
308}
309
310void intel_uncore_clear_discovery_tables(void)
311{
312 struct intel_uncore_discovery_type *type, *next;
313
314 rbtree_postorder_for_each_entry_safe(type, next, &discovery_tables, node) {
315 kfree(type->box_ctrl_die);
316 kfree(type);
317 }
318}
319
320DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
321DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
322DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
323DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
324DEFINE_UNCORE_FORMAT_ATTR(thresh, thresh, "config:24-31");
325
326static struct attribute *generic_uncore_formats_attr[] = {
327 &format_attr_event.attr,
328 &format_attr_umask.attr,
329 &format_attr_edge.attr,
330 &format_attr_inv.attr,
331 &format_attr_thresh.attr,
332 NULL,
333};
334
335static const struct attribute_group generic_uncore_format_group = {
336 .name = "format",
337 .attrs = generic_uncore_formats_attr,
338};
339
340static void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box)
341{
342 wrmsrl(uncore_msr_box_ctl(box), GENERIC_PMON_BOX_CTL_INT);
343}
344
345static void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box)
346{
347 wrmsrl(uncore_msr_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ);
348}
349
350static void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box)
351{
352 wrmsrl(uncore_msr_box_ctl(box), 0);
353}
354
355static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box,
356 struct perf_event *event)
357{
358 struct hw_perf_event *hwc = &event->hw;
359
360 wrmsrl(hwc->config_base, hwc->config);
361}
362
363static void intel_generic_uncore_msr_disable_event(struct intel_uncore_box *box,
364 struct perf_event *event)
365{
366 struct hw_perf_event *hwc = &event->hw;
367
368 wrmsrl(hwc->config_base, 0);
369}
370
371static struct intel_uncore_ops generic_uncore_msr_ops = {
372 .init_box = intel_generic_uncore_msr_init_box,
373 .disable_box = intel_generic_uncore_msr_disable_box,
374 .enable_box = intel_generic_uncore_msr_enable_box,
375 .disable_event = intel_generic_uncore_msr_disable_event,
376 .enable_event = intel_generic_uncore_msr_enable_event,
377 .read_counter = uncore_msr_read_counter,
378};
379
380static void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box)
381{
382 struct pci_dev *pdev = box->pci_dev;
383 int box_ctl = uncore_pci_box_ctl(box);
384
385 __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
386 pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_INT);
387}
388
389static void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box)
390{
391 struct pci_dev *pdev = box->pci_dev;
392 int box_ctl = uncore_pci_box_ctl(box);
393
394 pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_FRZ);
395}
396
397static void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box)
398{
399 struct pci_dev *pdev = box->pci_dev;
400 int box_ctl = uncore_pci_box_ctl(box);
401
402 pci_write_config_dword(pdev, box_ctl, 0);
403}
404
405static void intel_generic_uncore_pci_enable_event(struct intel_uncore_box *box,
406 struct perf_event *event)
407{
408 struct pci_dev *pdev = box->pci_dev;
409 struct hw_perf_event *hwc = &event->hw;
410
411 pci_write_config_dword(pdev, hwc->config_base, hwc->config);
412}
413
414static void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box,
415 struct perf_event *event)
416{
417 struct pci_dev *pdev = box->pci_dev;
418 struct hw_perf_event *hwc = &event->hw;
419
420 pci_write_config_dword(pdev, hwc->config_base, 0);
421}
422
423static u64 intel_generic_uncore_pci_read_counter(struct intel_uncore_box *box,
424 struct perf_event *event)
425{
426 struct pci_dev *pdev = box->pci_dev;
427 struct hw_perf_event *hwc = &event->hw;
428 u64 count = 0;
429
430 pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count);
431 pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1);
432
433 return count;
434}
435
436static struct intel_uncore_ops generic_uncore_pci_ops = {
437 .init_box = intel_generic_uncore_pci_init_box,
438 .disable_box = intel_generic_uncore_pci_disable_box,
439 .enable_box = intel_generic_uncore_pci_enable_box,
440 .disable_event = intel_generic_uncore_pci_disable_event,
441 .enable_event = intel_generic_uncore_pci_enable_event,
442 .read_counter = intel_generic_uncore_pci_read_counter,
443};
444
445#define UNCORE_GENERIC_MMIO_SIZE 0x4000
446
447static unsigned int generic_uncore_mmio_box_ctl(struct intel_uncore_box *box)
448{
449 struct intel_uncore_type *type = box->pmu->type;
450
451 if (!type->box_ctls || !type->box_ctls[box->dieid] || !type->mmio_offsets)
452 return 0;
453
454 return type->box_ctls[box->dieid] + type->mmio_offsets[box->pmu->pmu_idx];
455}
456
457static void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box)
458{
459 unsigned int box_ctl = generic_uncore_mmio_box_ctl(box);
460 struct intel_uncore_type *type = box->pmu->type;
461 resource_size_t addr;
462
463 if (!box_ctl) {
464 pr_warn("Uncore type %d box %d: Invalid box control address.\n",
465 type->type_id, type->box_ids[box->pmu->pmu_idx]);
466 return;
467 }
468
469 addr = box_ctl;
470 box->io_addr = ioremap(addr, UNCORE_GENERIC_MMIO_SIZE);
471 if (!box->io_addr) {
472 pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n",
473 type->type_id, type->box_ids[box->pmu->pmu_idx],
474 (unsigned long long)addr);
475 return;
476 }
477
478 writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr);
479}
480
481static void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box)
482{
483 if (!box->io_addr)
484 return;
485
486 writel(GENERIC_PMON_BOX_CTL_FRZ, box->io_addr);
487}
488
489static void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box)
490{
491 if (!box->io_addr)
492 return;
493
494 writel(0, box->io_addr);
495}
496
497static void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box,
498 struct perf_event *event)
499{
500 struct hw_perf_event *hwc = &event->hw;
501
502 if (!box->io_addr)
503 return;
504
505 writel(hwc->config, box->io_addr + hwc->config_base);
506}
507
508static void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box,
509 struct perf_event *event)
510{
511 struct hw_perf_event *hwc = &event->hw;
512
513 if (!box->io_addr)
514 return;
515
516 writel(0, box->io_addr + hwc->config_base);
517}
518
519static struct intel_uncore_ops generic_uncore_mmio_ops = {
520 .init_box = intel_generic_uncore_mmio_init_box,
521 .exit_box = uncore_mmio_exit_box,
522 .disable_box = intel_generic_uncore_mmio_disable_box,
523 .enable_box = intel_generic_uncore_mmio_enable_box,
524 .disable_event = intel_generic_uncore_mmio_disable_event,
525 .enable_event = intel_generic_uncore_mmio_enable_event,
526 .read_counter = uncore_mmio_read_counter,
527};
528
529static bool uncore_update_uncore_type(enum uncore_access_type type_id,
530 struct intel_uncore_type *uncore,
531 struct intel_uncore_discovery_type *type)
532{
533 uncore->type_id = type->type;
534 uncore->num_boxes = type->num_boxes;
535 uncore->num_counters = type->num_counters;
536 uncore->perf_ctr_bits = type->counter_width;
537 uncore->box_ids = type->ids;
538
539 switch (type_id) {
540 case UNCORE_ACCESS_MSR:
541 uncore->ops = &generic_uncore_msr_ops;
542 uncore->perf_ctr = (unsigned int)type->box_ctrl + type->ctr_offset;
543 uncore->event_ctl = (unsigned int)type->box_ctrl + type->ctl_offset;
544 uncore->box_ctl = (unsigned int)type->box_ctrl;
545 uncore->msr_offsets = type->box_offset;
546 break;
547 case UNCORE_ACCESS_PCI:
548 uncore->ops = &generic_uncore_pci_ops;
549 uncore->perf_ctr = (unsigned int)UNCORE_DISCOVERY_PCI_BOX_CTRL(type->box_ctrl) + type->ctr_offset;
550 uncore->event_ctl = (unsigned int)UNCORE_DISCOVERY_PCI_BOX_CTRL(type->box_ctrl) + type->ctl_offset;
551 uncore->box_ctl = (unsigned int)UNCORE_DISCOVERY_PCI_BOX_CTRL(type->box_ctrl);
552 uncore->box_ctls = type->box_ctrl_die;
553 uncore->pci_offsets = type->box_offset;
554 break;
555 case UNCORE_ACCESS_MMIO:
556 uncore->ops = &generic_uncore_mmio_ops;
557 uncore->perf_ctr = (unsigned int)type->ctr_offset;
558 uncore->event_ctl = (unsigned int)type->ctl_offset;
559 uncore->box_ctl = (unsigned int)type->box_ctrl;
560 uncore->box_ctls = type->box_ctrl_die;
561 uncore->mmio_offsets = type->box_offset;
562 uncore->mmio_map_size = UNCORE_GENERIC_MMIO_SIZE;
563 break;
564 default:
565 return false;
566 }
567
568 return true;
569}
570
571static struct intel_uncore_type **
572intel_uncore_generic_init_uncores(enum uncore_access_type type_id)
573{
574 struct intel_uncore_discovery_type *type;
575 struct intel_uncore_type **uncores;
576 struct intel_uncore_type *uncore;
577 struct rb_node *node;
578 int i = 0;
579
580 uncores = kcalloc(num_discovered_types[type_id] + 1,
581 sizeof(struct intel_uncore_type *), GFP_KERNEL);
582 if (!uncores)
583 return empty_uncore;
584
585 for (node = rb_first(&discovery_tables); node; node = rb_next(node)) {
586 type = rb_entry(node, struct intel_uncore_discovery_type, node);
587 if (type->access_type != type_id)
588 continue;
589
590 uncore = kzalloc(sizeof(struct intel_uncore_type), GFP_KERNEL);
591 if (!uncore)
592 break;
593
594 uncore->event_mask = GENERIC_PMON_RAW_EVENT_MASK;
595 uncore->format_group = &generic_uncore_format_group;
596
597 if (!uncore_update_uncore_type(type_id, uncore, type)) {
598 kfree(uncore);
599 continue;
600 }
601 uncores[i++] = uncore;
602 }
603
604 return uncores;
605}
606
607void intel_uncore_generic_uncore_cpu_init(void)
608{
609 uncore_msr_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MSR);
610}
611
612int intel_uncore_generic_uncore_pci_init(void)
613{
614 uncore_pci_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_PCI);
615
616 return 0;
617}
618
619void intel_uncore_generic_uncore_mmio_init(void)
620{
621 uncore_mmio_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MMIO);
622}
623