1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Synopsys DesignWare PCIe PMU driver
4 *
5 * Copyright (C) 2021-2023 Alibaba Inc.
6 */
7
8 #include <linux/bitfield.h>
9 #include <linux/bitops.h>
10 #include <linux/cpuhotplug.h>
11 #include <linux/cpumask.h>
12 #include <linux/device.h>
13 #include <linux/errno.h>
14 #include <linux/kernel.h>
15 #include <linux/list.h>
16 #include <linux/perf_event.h>
17 #include <linux/pci.h>
18 #include <linux/platform_device.h>
19 #include <linux/smp.h>
20 #include <linux/sysfs.h>
21 #include <linux/types.h>
22
23 #define DWC_PCIE_EVENT_CNT_CTL 0x8
24
25 /*
26 * Event Counter Data Select includes two parts:
27 * - 27-24: Group number(4-bit: 0..0x7)
28 * - 23-16: Event number(8-bit: 0..0x13) within the Group
29 *
30 * Put them together as in TRM.
31 */
32 #define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16)
33 #define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8)
34 #define DWC_PCIE_CNT_STATUS BIT(7)
35 #define DWC_PCIE_CNT_ENABLE GENMASK(4, 2)
36 #define DWC_PCIE_PER_EVENT_OFF 0x1
37 #define DWC_PCIE_PER_EVENT_ON 0x3
38 #define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0)
39 #define DWC_PCIE_EVENT_PER_CLEAR 0x1
40
41 #define DWC_PCIE_EVENT_CNT_DATA 0xC
42
43 #define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10
44 #define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24)
45 #define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8)
46 #define DWC_PCIE_DURATION_MANUAL_CTL 0x0
47 #define DWC_PCIE_DURATION_1MS 0x1
48 #define DWC_PCIE_DURATION_10MS 0x2
49 #define DWC_PCIE_DURATION_100MS 0x3
50 #define DWC_PCIE_DURATION_1S 0x4
51 #define DWC_PCIE_DURATION_2S 0x5
52 #define DWC_PCIE_DURATION_4S 0x6
53 #define DWC_PCIE_DURATION_4US 0xFF
54 #define DWC_PCIE_TIME_BASED_TIMER_START BIT(0)
55 #define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1
56
57 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14
58 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18
59
60 /* Event attributes */
61 #define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0)
62 #define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16)
63 #define DWC_PCIE_CONFIG_LANE GENMASK(27, 20)
64
65 #define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config)
66 #define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config)
67 #define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config)
68
69 enum dwc_pcie_event_type {
70 DWC_PCIE_TIME_BASE_EVENT,
71 DWC_PCIE_LANE_EVENT,
72 DWC_PCIE_EVENT_TYPE_MAX,
73 };
74
75 #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0)
76 #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0)
77
78 struct dwc_pcie_pmu {
79 struct pmu pmu;
80 struct pci_dev *pdev; /* Root Port device */
81 u16 ras_des_offset;
82 u32 nr_lanes;
83
84 struct hlist_node cpuhp_node;
85 struct perf_event *event[DWC_PCIE_EVENT_TYPE_MAX];
86 int on_cpu;
87 };
88
89 #define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu))
90
91 static int dwc_pcie_pmu_hp_state;
92 static struct list_head dwc_pcie_dev_info_head =
93 LIST_HEAD_INIT(dwc_pcie_dev_info_head);
94 static bool notify;
95
96 struct dwc_pcie_dev_info {
97 struct platform_device *plat_dev;
98 struct pci_dev *pdev;
99 struct list_head dev_node;
100 };
101
102 struct dwc_pcie_pmu_vsec_id {
103 u16 vendor_id;
104 u16 vsec_id;
105 u8 vsec_rev;
106 };
107
108 /*
109 * VSEC IDs are allocated by the vendor, so a given ID may mean different
110 * things to different vendors. See PCIe r6.0, sec 7.9.5.2.
111 */
112 static const struct dwc_pcie_pmu_vsec_id dwc_pcie_pmu_vsec_ids[] = {
113 { .vendor_id = PCI_VENDOR_ID_ALIBABA,
114 .vsec_id = 0x02, .vsec_rev = 0x4 },
115 { .vendor_id = PCI_VENDOR_ID_AMPERE,
116 .vsec_id = 0x02, .vsec_rev = 0x4 },
117 { .vendor_id = PCI_VENDOR_ID_QCOM,
118 .vsec_id = 0x02, .vsec_rev = 0x4 },
119 {} /* terminator */
120 };
121
cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)122 static ssize_t cpumask_show(struct device *dev,
123 struct device_attribute *attr,
124 char *buf)
125 {
126 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev));
127
128 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu));
129 }
130 static DEVICE_ATTR_RO(cpumask);
131
132 static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = {
133 &dev_attr_cpumask.attr,
134 NULL
135 };
136
137 static struct attribute_group dwc_pcie_cpumask_attr_group = {
138 .attrs = dwc_pcie_pmu_cpumask_attrs,
139 };
140
141 struct dwc_pcie_format_attr {
142 struct device_attribute attr;
143 u64 field;
144 int config;
145 };
146
147 PMU_FORMAT_ATTR(eventid, "config:0-15");
148 PMU_FORMAT_ATTR(type, "config:16-19");
149 PMU_FORMAT_ATTR(lane, "config:20-27");
150
151 static struct attribute *dwc_pcie_format_attrs[] = {
152 &format_attr_type.attr,
153 &format_attr_eventid.attr,
154 &format_attr_lane.attr,
155 NULL,
156 };
157
158 static struct attribute_group dwc_pcie_format_attrs_group = {
159 .name = "format",
160 .attrs = dwc_pcie_format_attrs,
161 };
162
163 struct dwc_pcie_event_attr {
164 struct device_attribute attr;
165 enum dwc_pcie_event_type type;
166 u16 eventid;
167 u8 lane;
168 };
169
dwc_pcie_event_show(struct device * dev,struct device_attribute * attr,char * buf)170 static ssize_t dwc_pcie_event_show(struct device *dev,
171 struct device_attribute *attr, char *buf)
172 {
173 struct dwc_pcie_event_attr *eattr;
174
175 eattr = container_of(attr, typeof(*eattr), attr);
176
177 if (eattr->type == DWC_PCIE_LANE_EVENT)
178 return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n",
179 eattr->eventid, eattr->type);
180 else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT)
181 return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n",
182 eattr->eventid, eattr->type);
183
184 return 0;
185 }
186
187 #define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \
188 (&((struct dwc_pcie_event_attr[]) {{ \
189 .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \
190 .type = _type, \
191 .eventid = _eventid, \
192 .lane = _lane, \
193 }})[0].attr.attr)
194
195 #define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \
196 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0)
197 #define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \
198 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0)
199
200 static struct attribute *dwc_pcie_pmu_time_event_attrs[] = {
201 /* Group #0 */
202 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00),
203 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01),
204 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02),
205 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03),
206 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04),
207 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05),
208 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06),
209 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07),
210 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x08),
211 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x09),
212
213 /* Group #1 */
214 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_pcie_tlp_data_payload, 0x20),
215 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_pcie_tlp_data_payload, 0x21),
216 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_ccix_tlp_data_payload, 0x22),
217 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_ccix_tlp_data_payload, 0x23),
218
219 /*
220 * Leave it to the user to specify the lane ID to avoid generating
221 * a list of hundreds of events.
222 */
223 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600),
224 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601),
225 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602),
226 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603),
227 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nullified_tlp, 0x604),
228 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nullified_tlp, 0x605),
229 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tlp, 0x606),
230 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700),
231 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701),
232 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702),
233 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703),
234 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704),
235 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705),
236 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706),
237 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707),
238 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708),
239 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709),
240 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A),
241 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B),
242 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C),
243 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F),
244 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710),
245 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711),
246 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712),
247 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713),
248 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714),
249 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715),
250 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716),
251 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717),
252 NULL
253 };
254
255 static const struct attribute_group dwc_pcie_event_attrs_group = {
256 .name = "events",
257 .attrs = dwc_pcie_pmu_time_event_attrs,
258 };
259
260 static const struct attribute_group *dwc_pcie_attr_groups[] = {
261 &dwc_pcie_event_attrs_group,
262 &dwc_pcie_format_attrs_group,
263 &dwc_pcie_cpumask_attr_group,
264 NULL
265 };
266
dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu * pcie_pmu,bool enable)267 static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu,
268 bool enable)
269 {
270 struct pci_dev *pdev = pcie_pmu->pdev;
271 u16 ras_des_offset = pcie_pmu->ras_des_offset;
272
273 if (enable)
274 pci_clear_and_set_config_dword(pdev,
275 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
276 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);
277 else
278 pci_clear_and_set_config_dword(pdev,
279 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
280 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF);
281 }
282
dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu * pcie_pmu,bool enable)283 static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu,
284 bool enable)
285 {
286 struct pci_dev *pdev = pcie_pmu->pdev;
287 u16 ras_des_offset = pcie_pmu->ras_des_offset;
288
289 pci_clear_and_set_config_dword(pdev,
290 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL,
291 DWC_PCIE_TIME_BASED_TIMER_START, enable);
292 }
293
dwc_pcie_pmu_read_lane_event_counter(struct perf_event * event)294 static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event)
295 {
296 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
297 struct pci_dev *pdev = pcie_pmu->pdev;
298 u16 ras_des_offset = pcie_pmu->ras_des_offset;
299 u32 val;
300
301 pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val);
302
303 return val;
304 }
305
dwc_pcie_pmu_read_time_based_counter(struct perf_event * event)306 static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event)
307 {
308 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
309 struct pci_dev *pdev = pcie_pmu->pdev;
310 int event_id = DWC_PCIE_EVENT_ID(event);
311 u16 ras_des_offset = pcie_pmu->ras_des_offset;
312 u32 lo, hi, ss;
313 u64 val;
314
315 /*
316 * The 64-bit value of the data counter is spread across two
317 * registers that are not synchronized. In order to read them
318 * atomically, ensure that the high 32 bits match before and after
319 * reading the low 32 bits.
320 */
321 pci_read_config_dword(pdev,
322 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi);
323 do {
324 /* snapshot the high 32 bits */
325 ss = hi;
326
327 pci_read_config_dword(
328 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW,
329 &lo);
330 pci_read_config_dword(
331 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH,
332 &hi);
333 } while (hi != ss);
334
335 val = ((u64)hi << 32) | lo;
336 /*
337 * The Group#1 event measures the amount of data processed in 16-byte
338 * units. Simplify the end-user interface by multiplying the counter
339 * at the point of read.
340 */
341 if (event_id >= 0x20 && event_id <= 0x23)
342 val *= 16;
343
344 return val;
345 }
346
dwc_pcie_pmu_event_update(struct perf_event * event)347 static void dwc_pcie_pmu_event_update(struct perf_event *event)
348 {
349 struct hw_perf_event *hwc = &event->hw;
350 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
351 u64 delta, prev, now = 0;
352
353 do {
354 prev = local64_read(&hwc->prev_count);
355
356 if (type == DWC_PCIE_LANE_EVENT)
357 now = dwc_pcie_pmu_read_lane_event_counter(event);
358 else if (type == DWC_PCIE_TIME_BASE_EVENT)
359 now = dwc_pcie_pmu_read_time_based_counter(event);
360
361 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
362
363 delta = (now - prev) & DWC_PCIE_MAX_PERIOD;
364 /* 32-bit counter for Lane Event Counting */
365 if (type == DWC_PCIE_LANE_EVENT)
366 delta &= DWC_PCIE_LANE_EVENT_MAX_PERIOD;
367
368 local64_add(delta, &event->count);
369 }
370
dwc_pcie_pmu_event_init(struct perf_event * event)371 static int dwc_pcie_pmu_event_init(struct perf_event *event)
372 {
373 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
374 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
375 struct perf_event *sibling;
376 u32 lane;
377
378 if (event->attr.type != event->pmu->type)
379 return -ENOENT;
380
381 /* We don't support sampling */
382 if (is_sampling_event(event))
383 return -EINVAL;
384
385 /* We cannot support task bound events */
386 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK)
387 return -EINVAL;
388
389 if (event->group_leader != event &&
390 !is_software_event(event->group_leader))
391 return -EINVAL;
392
393 for_each_sibling_event(sibling, event->group_leader) {
394 if (sibling->pmu != event->pmu && !is_software_event(sibling))
395 return -EINVAL;
396 }
397
398 if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX)
399 return -EINVAL;
400
401 if (type == DWC_PCIE_LANE_EVENT) {
402 lane = DWC_PCIE_EVENT_LANE(event);
403 if (lane < 0 || lane >= pcie_pmu->nr_lanes)
404 return -EINVAL;
405 }
406
407 event->cpu = pcie_pmu->on_cpu;
408
409 return 0;
410 }
411
dwc_pcie_pmu_event_start(struct perf_event * event,int flags)412 static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags)
413 {
414 struct hw_perf_event *hwc = &event->hw;
415 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
416 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
417
418 hwc->state = 0;
419 local64_set(&hwc->prev_count, 0);
420
421 if (type == DWC_PCIE_LANE_EVENT)
422 dwc_pcie_pmu_lane_event_enable(pcie_pmu, true);
423 else if (type == DWC_PCIE_TIME_BASE_EVENT)
424 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true);
425 }
426
dwc_pcie_pmu_event_stop(struct perf_event * event,int flags)427 static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags)
428 {
429 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
430 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
431 struct hw_perf_event *hwc = &event->hw;
432
433 if (event->hw.state & PERF_HES_STOPPED)
434 return;
435
436 if (type == DWC_PCIE_LANE_EVENT)
437 dwc_pcie_pmu_lane_event_enable(pcie_pmu, false);
438 else if (type == DWC_PCIE_TIME_BASE_EVENT)
439 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false);
440
441 dwc_pcie_pmu_event_update(event);
442 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
443 }
444
dwc_pcie_pmu_event_add(struct perf_event * event,int flags)445 static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags)
446 {
447 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
448 struct pci_dev *pdev = pcie_pmu->pdev;
449 struct hw_perf_event *hwc = &event->hw;
450 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
451 int event_id = DWC_PCIE_EVENT_ID(event);
452 int lane = DWC_PCIE_EVENT_LANE(event);
453 u16 ras_des_offset = pcie_pmu->ras_des_offset;
454 u32 ctrl;
455
456 /* one counter for each type and it is in use */
457 if (pcie_pmu->event[type])
458 return -ENOSPC;
459
460 pcie_pmu->event[type] = event;
461 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
462
463 if (type == DWC_PCIE_LANE_EVENT) {
464 /* EVENT_COUNTER_DATA_REG needs clear manually */
465 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
466 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
467 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) |
468 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
469 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
470 ctrl);
471 } else if (type == DWC_PCIE_TIME_BASE_EVENT) {
472 /*
473 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely
474 * use it with any manually controlled duration. And it is
475 * cleared when next measurement starts.
476 */
477 ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) |
478 FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL,
479 DWC_PCIE_DURATION_MANUAL_CTL) |
480 DWC_PCIE_TIME_BASED_CNT_ENABLE;
481 pci_write_config_dword(
482 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl);
483 }
484
485 if (flags & PERF_EF_START)
486 dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD);
487
488 perf_event_update_userpage(event);
489
490 return 0;
491 }
492
dwc_pcie_pmu_event_del(struct perf_event * event,int flags)493 static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags)
494 {
495 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
496 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
497
498 dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE);
499 perf_event_update_userpage(event);
500 pcie_pmu->event[type] = NULL;
501 }
502
dwc_pcie_pmu_remove_cpuhp_instance(void * hotplug_node)503 static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node)
504 {
505 cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node);
506 }
507
508 /*
509 * Find the binded DES capability device info of a PCI device.
510 * @pdev: The PCI device.
511 */
dwc_pcie_find_dev_info(struct pci_dev * pdev)512 static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev)
513 {
514 struct dwc_pcie_dev_info *dev_info;
515
516 list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node)
517 if (dev_info->pdev == pdev)
518 return dev_info;
519
520 return NULL;
521 }
522
dwc_pcie_unregister_pmu(void * data)523 static void dwc_pcie_unregister_pmu(void *data)
524 {
525 struct dwc_pcie_pmu *pcie_pmu = data;
526
527 perf_pmu_unregister(&pcie_pmu->pmu);
528 }
529
dwc_pcie_des_cap(struct pci_dev * pdev)530 static u16 dwc_pcie_des_cap(struct pci_dev *pdev)
531 {
532 const struct dwc_pcie_pmu_vsec_id *vid;
533 u16 vsec;
534 u32 val;
535
536 if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT))
537 return 0;
538
539 for (vid = dwc_pcie_pmu_vsec_ids; vid->vendor_id; vid++) {
540 vsec = pci_find_vsec_capability(pdev, vid->vendor_id,
541 vid->vsec_id);
542 if (vsec) {
543 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER,
544 &val);
545 if (PCI_VNDR_HEADER_REV(val) == vid->vsec_rev) {
546 pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability RAS DES\n");
547 return vsec;
548 }
549 }
550 }
551 return 0;
552 }
553
dwc_pcie_unregister_dev(struct dwc_pcie_dev_info * dev_info)554 static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info)
555 {
556 platform_device_unregister(dev_info->plat_dev);
557 list_del(&dev_info->dev_node);
558 kfree(dev_info);
559 }
560
dwc_pcie_register_dev(struct pci_dev * pdev)561 static int dwc_pcie_register_dev(struct pci_dev *pdev)
562 {
563 struct platform_device *plat_dev;
564 struct dwc_pcie_dev_info *dev_info;
565 u32 sbdf;
566
567 sbdf = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn);
568 plat_dev = platform_device_register_simple("dwc_pcie_pmu", sbdf, NULL, 0);
569 if (IS_ERR(plat_dev))
570 return PTR_ERR(plat_dev);
571
572 dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL);
573 if (!dev_info) {
574 platform_device_unregister(plat_dev);
575 return -ENOMEM;
576 }
577
578 /* Cache platform device to handle pci device hotplug */
579 dev_info->plat_dev = plat_dev;
580 dev_info->pdev = pdev;
581 list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head);
582
583 return 0;
584 }
585
dwc_pcie_pmu_notifier(struct notifier_block * nb,unsigned long action,void * data)586 static int dwc_pcie_pmu_notifier(struct notifier_block *nb,
587 unsigned long action, void *data)
588 {
589 struct device *dev = data;
590 struct pci_dev *pdev = to_pci_dev(dev);
591 struct dwc_pcie_dev_info *dev_info;
592
593 switch (action) {
594 case BUS_NOTIFY_ADD_DEVICE:
595 if (!dwc_pcie_des_cap(pdev))
596 return NOTIFY_DONE;
597 if (dwc_pcie_register_dev(pdev))
598 return NOTIFY_BAD;
599 break;
600 case BUS_NOTIFY_DEL_DEVICE:
601 dev_info = dwc_pcie_find_dev_info(pdev);
602 if (!dev_info)
603 return NOTIFY_DONE;
604 dwc_pcie_unregister_dev(dev_info);
605 break;
606 }
607
608 return NOTIFY_OK;
609 }
610
611 static struct notifier_block dwc_pcie_pmu_nb = {
612 .notifier_call = dwc_pcie_pmu_notifier,
613 };
614
dwc_pcie_pmu_probe(struct platform_device * plat_dev)615 static int dwc_pcie_pmu_probe(struct platform_device *plat_dev)
616 {
617 struct pci_dev *pdev;
618 struct dwc_pcie_pmu *pcie_pmu;
619 char *name;
620 u32 sbdf;
621 u16 vsec;
622 int ret;
623
624 sbdf = plat_dev->id;
625 pdev = pci_get_domain_bus_and_slot(sbdf >> 16, PCI_BUS_NUM(sbdf & 0xffff),
626 sbdf & 0xff);
627 if (!pdev) {
628 pr_err("No pdev found for the sbdf 0x%x\n", sbdf);
629 return -ENODEV;
630 }
631
632 vsec = dwc_pcie_des_cap(pdev);
633 if (!vsec)
634 return -ENODEV;
635
636 pci_dev_put(pdev);
637 name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", sbdf);
638 if (!name)
639 return -ENOMEM;
640
641 pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL);
642 if (!pcie_pmu)
643 return -ENOMEM;
644
645 pcie_pmu->pdev = pdev;
646 pcie_pmu->ras_des_offset = vsec;
647 pcie_pmu->nr_lanes = pcie_get_width_cap(pdev);
648 pcie_pmu->on_cpu = -1;
649 pcie_pmu->pmu = (struct pmu){
650 .name = name,
651 .parent = &plat_dev->dev,
652 .module = THIS_MODULE,
653 .attr_groups = dwc_pcie_attr_groups,
654 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
655 .task_ctx_nr = perf_invalid_context,
656 .event_init = dwc_pcie_pmu_event_init,
657 .add = dwc_pcie_pmu_event_add,
658 .del = dwc_pcie_pmu_event_del,
659 .start = dwc_pcie_pmu_event_start,
660 .stop = dwc_pcie_pmu_event_stop,
661 .read = dwc_pcie_pmu_event_update,
662 };
663
664 /* Add this instance to the list used by the offline callback */
665 ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state,
666 &pcie_pmu->cpuhp_node);
667 if (ret) {
668 pci_err(pdev, "Error %d registering hotplug @%x\n", ret, sbdf);
669 return ret;
670 }
671
672 /* Unwind when platform driver removes */
673 ret = devm_add_action_or_reset(&plat_dev->dev,
674 dwc_pcie_pmu_remove_cpuhp_instance,
675 &pcie_pmu->cpuhp_node);
676 if (ret)
677 return ret;
678
679 ret = perf_pmu_register(&pcie_pmu->pmu, name, -1);
680 if (ret) {
681 pci_err(pdev, "Error %d registering PMU @%x\n", ret, sbdf);
682 return ret;
683 }
684 ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu,
685 pcie_pmu);
686 if (ret)
687 return ret;
688
689 return 0;
690 }
691
dwc_pcie_pmu_online_cpu(unsigned int cpu,struct hlist_node * cpuhp_node)692 static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
693 {
694 struct dwc_pcie_pmu *pcie_pmu;
695
696 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
697 if (pcie_pmu->on_cpu == -1)
698 pcie_pmu->on_cpu = cpumask_local_spread(
699 0, dev_to_node(&pcie_pmu->pdev->dev));
700
701 return 0;
702 }
703
dwc_pcie_pmu_offline_cpu(unsigned int cpu,struct hlist_node * cpuhp_node)704 static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
705 {
706 struct dwc_pcie_pmu *pcie_pmu;
707 struct pci_dev *pdev;
708 unsigned int target;
709 int node;
710
711 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
712 /* Nothing to do if this CPU doesn't own the PMU */
713 if (cpu != pcie_pmu->on_cpu)
714 return 0;
715
716 pcie_pmu->on_cpu = -1;
717 pdev = pcie_pmu->pdev;
718 node = dev_to_node(&pdev->dev);
719
720 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
721 if (target >= nr_cpu_ids)
722 target = cpumask_any_but(cpu_online_mask, cpu);
723
724 if (target >= nr_cpu_ids) {
725 pci_err(pdev, "There is no CPU to set\n");
726 return 0;
727 }
728
729 /* This PMU does NOT support interrupt, just migrate context. */
730 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target);
731 pcie_pmu->on_cpu = target;
732
733 return 0;
734 }
735
736 static struct platform_driver dwc_pcie_pmu_driver = {
737 .probe = dwc_pcie_pmu_probe,
738 .driver = {.name = "dwc_pcie_pmu",},
739 };
740
dwc_pcie_cleanup_devices(void)741 static void dwc_pcie_cleanup_devices(void)
742 {
743 struct dwc_pcie_dev_info *dev_info, *tmp;
744
745 list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) {
746 dwc_pcie_unregister_dev(dev_info);
747 }
748 }
749
dwc_pcie_pmu_init(void)750 static int __init dwc_pcie_pmu_init(void)
751 {
752 struct pci_dev *pdev = NULL;
753 int ret;
754
755 for_each_pci_dev(pdev) {
756 if (!dwc_pcie_des_cap(pdev))
757 continue;
758
759 ret = dwc_pcie_register_dev(pdev);
760 if (ret) {
761 pci_dev_put(pdev);
762 goto err_cleanup;
763 }
764 }
765
766 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
767 "perf/dwc_pcie_pmu:online",
768 dwc_pcie_pmu_online_cpu,
769 dwc_pcie_pmu_offline_cpu);
770 if (ret < 0)
771 goto err_cleanup;
772
773 dwc_pcie_pmu_hp_state = ret;
774
775 ret = platform_driver_register(&dwc_pcie_pmu_driver);
776 if (ret)
777 goto err_remove_cpuhp;
778
779 ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
780 if (ret)
781 goto err_unregister_driver;
782 notify = true;
783
784 return 0;
785
786 err_unregister_driver:
787 platform_driver_unregister(&dwc_pcie_pmu_driver);
788 err_remove_cpuhp:
789 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
790 err_cleanup:
791 dwc_pcie_cleanup_devices();
792 return ret;
793 }
794
dwc_pcie_pmu_exit(void)795 static void __exit dwc_pcie_pmu_exit(void)
796 {
797 if (notify)
798 bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
799 dwc_pcie_cleanup_devices();
800 platform_driver_unregister(&dwc_pcie_pmu_driver);
801 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
802 }
803
804 module_init(dwc_pcie_pmu_init);
805 module_exit(dwc_pcie_pmu_exit);
806
807 MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller");
808 MODULE_AUTHOR("Shuai Xue <[email protected]>");
809 MODULE_LICENSE("GPL v2");
810