59 #define to_ddr_pmu(p)		container_of(p, struct ddr_pmu, pmu)
79 	const char *identifier;		/* system PMU identifier for userspace */
84 	struct pmu pmu;  member
112 static inline bool axi_filter_v1(struct ddr_pmu *pmu)  in axi_filter_v1()  argument
114 	return pmu->devtype_data->filter_ver == DDR_PERF_AXI_FILTER_V1;  in axi_filter_v1()
117 static inline bool axi_filter_v2(struct ddr_pmu *pmu)  in axi_filter_v2()  argument
119 	return pmu->devtype_data->filter_ver == DDR_PERF_AXI_FILTER_V2;  in axi_filter_v2()
123 	{ .compatible = "fsl,imx91-ddr-pmu", .data = &imx91_devtype_data },
124 	{ .compatible = "fsl,imx93-ddr-pmu", .data = &imx93_devtype_data },
125 	{ .compatible = "fsl,imx95-ddr-pmu", .data = &imx95_devtype_data },
134 	struct ddr_pmu *pmu = dev_get_drvdata(dev);  in ddr_perf_identifier_show()  local
136 	return sysfs_emit(page, "%s\n", pmu->devtype_data->identifier);  in ddr_perf_identifier_show()
154 	struct ddr_pmu *pmu = dev_get_drvdata(dev);  in ddr_perf_cpumask_show()  local
156 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));  in ddr_perf_cpumask_show()
317 	struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));  in ddr_perf_events_attrs_is_visible()  local
318 	struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);  in ddr_perf_events_attrs_is_visible()
365 static void ddr_perf_clear_counter(struct ddr_pmu *pmu, int counter)  in ddr_perf_clear_counter()  argument
368 		writel(0, pmu->base + PMC(counter) + 0x4);  in ddr_perf_clear_counter()
369 		writel(0, pmu->base + PMC(counter));  in ddr_perf_clear_counter()
371 		writel(0, pmu->base + PMC(counter));  in ddr_perf_clear_counter()
375 static u64 ddr_perf_read_counter(struct ddr_pmu *pmu, int counter)  in ddr_perf_read_counter()  argument
381 		val = readl_relaxed(pmu->base + PMC(counter));  in ddr_perf_read_counter()
387 		val_upper = readl_relaxed(pmu->base + PMC(counter) + 0x4);  in ddr_perf_read_counter()
388 		val_lower = readl_relaxed(pmu->base + PMC(counter));  in ddr_perf_read_counter()
389 	} while (val_upper != readl_relaxed(pmu->base + PMC(counter) + 0x4));  in ddr_perf_read_counter()
398 static void ddr_perf_counter_global_config(struct ddr_pmu *pmu, bool enable)  in ddr_perf_counter_global_config()  argument
402 	ctrl = readl_relaxed(pmu->base + PMGC0);  in ddr_perf_counter_global_config()
416 		writel(ctrl, pmu->base + PMGC0);  in ddr_perf_counter_global_config()
424 		writel(ctrl, pmu->base + PMGC0);  in ddr_perf_counter_global_config()
428 		writel(ctrl, pmu->base + PMGC0);  in ddr_perf_counter_global_config()
432 static void ddr_perf_counter_local_config(struct ddr_pmu *pmu, int config,  in ddr_perf_counter_local_config()  argument
438 	ctrl_a = readl_relaxed(pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
443 		writel(ctrl_a, pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
445 		ddr_perf_clear_counter(pmu, counter);  in ddr_perf_counter_local_config()
452 		writel(ctrl_a, pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
456 		writel(ctrl_a, pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
460 static void imx93_ddr_perf_monitor_config(struct ddr_pmu *pmu, int event,  in imx93_ddr_perf_monitor_config()  argument
468 	pmcfg1 = readl_relaxed(pmu->base + PMCFG1);  in imx93_ddr_perf_monitor_config()
476 	writel_relaxed(pmcfg1, pmu->base + PMCFG1);  in imx93_ddr_perf_monitor_config()
478 	pmcfg2 = readl_relaxed(pmu->base + PMCFG2);  in imx93_ddr_perf_monitor_config()
481 	writel_relaxed(pmcfg2, pmu->base + PMCFG2);  in imx93_ddr_perf_monitor_config()
484 static void imx95_ddr_perf_monitor_config(struct ddr_pmu *pmu, int event,  in imx95_ddr_perf_monitor_config()  argument
489 	pmcfg1 = readl_relaxed(pmu->base + PMCFG1);  in imx95_ddr_perf_monitor_config()
523 	writel_relaxed(pmcfg1, pmu->base + PMCFG1);  in imx95_ddr_perf_monitor_config()
526 		pmcfg = readl_relaxed(pmu->base + offset);  in imx95_ddr_perf_monitor_config()
531 		writel_relaxed(pmcfg, pmu->base + offset);  in imx95_ddr_perf_monitor_config()
537 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_update()  local
542 	new_raw_count = ddr_perf_read_counter(pmu, counter);  in ddr_perf_event_update()
546 	ddr_perf_clear_counter(pmu, counter);  in ddr_perf_event_update()
551 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_init()  local
555 	if (event->attr.type != event->pmu->type)  in ddr_perf_event_init()
562 		dev_warn(pmu->dev, "Can't provide per-task data!\n");  in ddr_perf_event_init()
571 	if (event->group_leader->pmu != event->pmu &&  in ddr_perf_event_init()
576 		if (sibling->pmu != event->pmu &&  in ddr_perf_event_init()
581 	event->cpu = pmu->cpu;  in ddr_perf_event_init()
589 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_start()  local
595 	ddr_perf_counter_local_config(pmu, event->attr.config, counter, true);  in ddr_perf_event_start()
599 static int ddr_perf_alloc_counter(struct ddr_pmu *pmu, int event, int counter)  in ddr_perf_alloc_counter()  argument
605 		if (pmu->events[CYCLES_COUNTER] == NULL)  in ddr_perf_alloc_counter()
609 		if (pmu->events[counter] == NULL)  in ddr_perf_alloc_counter()
614 			if (pmu->events[i] == NULL)  in ddr_perf_alloc_counter()
623 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_add()  local
633 	counter = ddr_perf_alloc_counter(pmu, event_id, counter);  in ddr_perf_event_add()
635 		dev_dbg(pmu->dev, "There are not enough counters\n");  in ddr_perf_event_add()
639 	pmu->events[counter] = event;  in ddr_perf_event_add()
640 	pmu->active_events++;  in ddr_perf_event_add()
644 	if (axi_filter_v1(pmu))  in ddr_perf_event_add()
646 		imx93_ddr_perf_monitor_config(pmu, event_id, counter, cfg1, cfg2);  in ddr_perf_event_add()
648 	if (axi_filter_v2(pmu))  in ddr_perf_event_add()
650 		imx95_ddr_perf_monitor_config(pmu, event_id, counter, cfg1, cfg2);  in ddr_perf_event_add()
660 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_stop()  local
664 	ddr_perf_counter_local_config(pmu, event->attr.config, counter, false);  in ddr_perf_event_stop()
672 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_del()  local
678 	pmu->events[counter] = NULL;  in ddr_perf_event_del()
679 	pmu->active_events--;  in ddr_perf_event_del()
683 static void ddr_perf_pmu_enable(struct pmu *pmu)  in ddr_perf_pmu_enable()  argument
685 	struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);  in ddr_perf_pmu_enable()
690 static void ddr_perf_pmu_disable(struct pmu *pmu)  in ddr_perf_pmu_disable()  argument
692 	struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);  in ddr_perf_pmu_disable()
697 static void ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base,  in ddr_perf_init()  argument
700 	*pmu = (struct ddr_pmu) {  in ddr_perf_init()
701 		.pmu = (struct pmu) {  in ddr_perf_init()
722 	struct ddr_pmu *pmu = (struct ddr_pmu *)p;  in ddr_perf_irq_handler()  local
738 		if (!pmu->events[i])  in ddr_perf_irq_handler()
741 		event = pmu->events[i];  in ddr_perf_irq_handler()
746 	ddr_perf_counter_global_config(pmu, true);  in ddr_perf_irq_handler()
753 	struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);  in ddr_perf_offline_cpu()  local
756 	if (cpu != pmu->cpu)  in ddr_perf_offline_cpu()
763 	perf_pmu_migrate_context(&pmu->pmu, cpu, target);  in ddr_perf_offline_cpu()
764 	pmu->cpu = target;  in ddr_perf_offline_cpu()
766 	WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)));  in ddr_perf_offline_cpu()
773 	struct ddr_pmu *pmu;  in ddr_perf_probe()  local
782 	pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);  in ddr_perf_probe()
783 	if (!pmu)  in ddr_perf_probe()
786 	ddr_perf_init(pmu, base, &pdev->dev);  in ddr_perf_probe()
788 	pmu->devtype_data = of_device_get_match_data(&pdev->dev);  in ddr_perf_probe()
790 	platform_set_drvdata(pdev, pmu);  in ddr_perf_probe()
792 	pmu->id = ida_alloc(&ddr_ida, GFP_KERNEL);  in ddr_perf_probe()
793 	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d", pmu->id);  in ddr_perf_probe()
799 	pmu->cpu = raw_smp_processor_id();  in ddr_perf_probe()
806 	pmu->cpuhp_state = ret;  in ddr_perf_probe()
808 	/* Register the pmu instance for cpu hotplug */  in ddr_perf_probe()
809 	ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_probe()
824 			       DDR_CPUHP_CB_NAME, pmu);  in ddr_perf_probe()
830 	pmu->irq = irq;  in ddr_perf_probe()
831 	ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu));  in ddr_perf_probe()
833 		dev_err(pmu->dev, "Failed to set interrupt affinity\n");  in ddr_perf_probe()
837 	ret = perf_pmu_register(&pmu->pmu, name, -1);  in ddr_perf_probe()
844 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_probe()
846 	cpuhp_remove_multi_state(pmu->cpuhp_state);  in ddr_perf_probe()
849 	ida_free(&ddr_ida, pmu->id);  in ddr_perf_probe()
850 	dev_warn(&pdev->dev, "i.MX9 DDR Perf PMU failed (%d), disabled\n", ret);  in ddr_perf_probe()
856 	struct ddr_pmu *pmu = platform_get_drvdata(pdev);  in ddr_perf_remove()  local
858 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_remove()
859 	cpuhp_remove_multi_state(pmu->cpuhp_state);  in ddr_perf_remove()
861 	perf_pmu_unregister(&pmu->pmu);  in ddr_perf_remove()
863 	ida_free(&ddr_ida, pmu->id);  in ddr_perf_remove()
868 		.name                = "imx9-ddr-pmu",