5  * Based on arch/x86/kernel/cpu/intel_cacheinfo.c
14 #include <linux/cpu.h>
23 /* pointer to per cpu cacheinfo */
25 #define ci_cacheinfo(cpu)	(&per_cpu(ci_cpu_cacheinfo, cpu))  argument
26 #define cache_leaves(cpu)	(ci_cacheinfo(cpu)->num_leaves)  argument
27 #define per_cpu_cacheinfo(cpu)	(ci_cacheinfo(cpu)->info_list)  argument
28 #define per_cpu_cacheinfo_idx(cpu, idx)		\  argument
29 				(per_cpu_cacheinfo(cpu) + (idx))
34 struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu)  in get_cpu_cacheinfo()  argument
36 	return ci_cacheinfo(cpu);  in get_cpu_cacheinfo()
57 bool last_level_cache_is_valid(unsigned int cpu)  in last_level_cache_is_valid()  argument
61 	if (!cache_leaves(cpu) || !per_cpu_cacheinfo(cpu))  in last_level_cache_is_valid()
64 	llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);  in last_level_cache_is_valid()
203 static int cache_setup_of_node(unsigned int cpu)  in cache_setup_of_node()  argument
208 	struct device_node *np __free(device_node) = of_cpu_device_node_get(cpu);  in cache_setup_of_node()
210 		pr_err("Failed to find cpu%d device node\n", cpu);  in cache_setup_of_node()
218 	while (index < cache_leaves(cpu)) {  in cache_setup_of_node()
219 		this_leaf = per_cpu_cacheinfo_idx(cpu, index);  in cache_setup_of_node()
231 	if (index != cache_leaves(cpu)) /* not all OF nodes populated */  in cache_setup_of_node()
277 int init_of_cache_level(unsigned int cpu)  in init_of_cache_level()  argument
279 	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);  in init_of_cache_level()
280 	struct device_node *np __free(device_node) = of_cpu_device_node_get(cpu);  in init_of_cache_level()
315 static inline int cache_setup_of_node(unsigned int cpu) { return 0; }  in cache_setup_of_node()  argument
316 int init_of_cache_level(unsigned int cpu) { return 0; }  in init_of_cache_level()  argument
319 int __weak cache_setup_acpi(unsigned int cpu)  in cache_setup_acpi()  argument
326 static int cache_setup_properties(unsigned int cpu)  in cache_setup_properties()  argument
331 		ret = cache_setup_of_node(cpu);  in cache_setup_properties()
333 		ret = cache_setup_acpi(cpu);  in cache_setup_properties()
342 static int cache_shared_cpu_map_setup(unsigned int cpu)  in cache_shared_cpu_map_setup()  argument
344 	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);  in cache_shared_cpu_map_setup()
357 	if (!last_level_cache_is_valid(cpu) && !use_arch_info) {  in cache_shared_cpu_map_setup()
358 		ret = cache_setup_properties(cpu);  in cache_shared_cpu_map_setup()
363 	for (index = 0; index < cache_leaves(cpu); index++) {  in cache_shared_cpu_map_setup()
366 		this_leaf = per_cpu_cacheinfo_idx(cpu, index);  in cache_shared_cpu_map_setup()
368 		cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map);  in cache_shared_cpu_map_setup()
370 			if (i == cpu || !per_cpu_cacheinfo(i))  in cache_shared_cpu_map_setup()
385 					cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map);  in cache_shared_cpu_map_setup()
396 	/* shared_cpu_map is now populated for the cpu */  in cache_shared_cpu_map_setup()
401 static void cache_shared_cpu_map_remove(unsigned int cpu)  in cache_shared_cpu_map_remove()  argument
403 	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);  in cache_shared_cpu_map_remove()
407 	for (index = 0; index < cache_leaves(cpu); index++) {  in cache_shared_cpu_map_remove()
408 		this_leaf = per_cpu_cacheinfo_idx(cpu, index);  in cache_shared_cpu_map_remove()
410 			if (sibling == cpu || !per_cpu_cacheinfo(sibling))  in cache_shared_cpu_map_remove()
426 					cpumask_clear_cpu(cpu, &sib_leaf->shared_cpu_map);  in cache_shared_cpu_map_remove()
434 	/* cpu is no longer populated in the shared map */  in cache_shared_cpu_map_remove()
438 static void free_cache_attributes(unsigned int cpu)  in free_cache_attributes()  argument
440 	if (!per_cpu_cacheinfo(cpu))  in free_cache_attributes()
443 	cache_shared_cpu_map_remove(cpu);  in free_cache_attributes()
446 int __weak early_cache_level(unsigned int cpu)  in early_cache_level()  argument
451 int __weak init_cache_level(unsigned int cpu)  in init_cache_level()  argument
456 int __weak populate_cache_leaves(unsigned int cpu)  in populate_cache_leaves()  argument
461 static inline int allocate_cache_info(int cpu)  in allocate_cache_info()  argument
463 	per_cpu_cacheinfo(cpu) = kcalloc(cache_leaves(cpu), sizeof(struct cacheinfo), GFP_ATOMIC);  in allocate_cache_info()
464 	if (!per_cpu_cacheinfo(cpu)) {  in allocate_cache_info()
465 		cache_leaves(cpu) = 0;  in allocate_cache_info()
472 int fetch_cache_info(unsigned int cpu)  in fetch_cache_info()  argument
474 	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);  in fetch_cache_info()
479 		ret = init_of_cache_level(cpu);  in fetch_cache_info()
481 		ret = acpi_get_cache_info(cpu, &levels, &split_levels);  in fetch_cache_info()
494 	if (ret || !cache_leaves(cpu)) {  in fetch_cache_info()
495 		ret = early_cache_level(cpu);  in fetch_cache_info()
499 		if (!cache_leaves(cpu))  in fetch_cache_info()
505 	return allocate_cache_info(cpu);  in fetch_cache_info()
508 static inline int init_level_allocate_ci(unsigned int cpu)  in init_level_allocate_ci()  argument
510 	unsigned int early_leaves = cache_leaves(cpu);  in init_level_allocate_ci()
513 	 * via fetch_cache_info() and this also gets called as CPU hotplug  in init_level_allocate_ci()
522 	if (per_cpu_cacheinfo(cpu) && !ci_cacheinfo(cpu)->early_ci_levels)  in init_level_allocate_ci()
525 	if (init_cache_level(cpu) || !cache_leaves(cpu))  in init_level_allocate_ci()
531 	 * (e.g. as CPU hotplug callbacks).  in init_level_allocate_ci()
533 	ci_cacheinfo(cpu)->early_ci_levels = false;  in init_level_allocate_ci()
539 	if (cache_leaves(cpu) <= early_leaves && per_cpu_cacheinfo(cpu))  in init_level_allocate_ci()
542 	kfree(per_cpu_cacheinfo(cpu));  in init_level_allocate_ci()
543 	return allocate_cache_info(cpu);  in init_level_allocate_ci()
546 int detect_cache_attributes(unsigned int cpu)  in detect_cache_attributes()  argument
550 	ret = init_level_allocate_ci(cpu);  in detect_cache_attributes()
556 	 * update the cpu map.  in detect_cache_attributes()
558 	if (!last_level_cache_is_valid(cpu)) {  in detect_cache_attributes()
563 		ret = populate_cache_leaves(cpu);  in detect_cache_attributes()
573 	ret = cache_shared_cpu_map_setup(cpu);  in detect_cache_attributes()
575 		pr_warn("Unable to detect cache hierarchy for CPU %d\n", cpu);  in detect_cache_attributes()
582 	free_cache_attributes(cpu);  in detect_cache_attributes()
588 #define per_cpu_cache_dev(cpu)	(per_cpu(ci_cache_dev, cpu))  argument
594 #define per_cpu_index_dev(cpu)	(per_cpu(ci_index_dev, cpu))  argument
595 #define per_cache_index_dev(cpu, idx)	((per_cpu_index_dev(cpu))[idx])  argument
803 /* Add/Remove cache interface for CPU device */
804 static void cpu_cache_sysfs_exit(unsigned int cpu)  in cpu_cache_sysfs_exit()  argument
809 	if (per_cpu_index_dev(cpu)) {  in cpu_cache_sysfs_exit()
810 		for (i = 0; i < cache_leaves(cpu); i++) {  in cpu_cache_sysfs_exit()
811 			ci_dev = per_cache_index_dev(cpu, i);  in cpu_cache_sysfs_exit()
816 		kfree(per_cpu_index_dev(cpu));  in cpu_cache_sysfs_exit()
817 		per_cpu_index_dev(cpu) = NULL;  in cpu_cache_sysfs_exit()
819 	device_unregister(per_cpu_cache_dev(cpu));  in cpu_cache_sysfs_exit()
820 	per_cpu_cache_dev(cpu) = NULL;  in cpu_cache_sysfs_exit()
823 static int cpu_cache_sysfs_init(unsigned int cpu)  in cpu_cache_sysfs_init()  argument
825 	struct device *dev = get_cpu_device(cpu);  in cpu_cache_sysfs_init()
827 	if (per_cpu_cacheinfo(cpu) == NULL)  in cpu_cache_sysfs_init()
830 	per_cpu_cache_dev(cpu) = cpu_device_create(dev, NULL, NULL, "cache");  in cpu_cache_sysfs_init()
831 	if (IS_ERR(per_cpu_cache_dev(cpu)))  in cpu_cache_sysfs_init()
832 		return PTR_ERR(per_cpu_cache_dev(cpu));  in cpu_cache_sysfs_init()
835 	per_cpu_index_dev(cpu) = kcalloc(cache_leaves(cpu),  in cpu_cache_sysfs_init()
837 	if (unlikely(per_cpu_index_dev(cpu) == NULL))  in cpu_cache_sysfs_init()
843 	cpu_cache_sysfs_exit(cpu);  in cpu_cache_sysfs_init()
847 static int cache_add_dev(unsigned int cpu)  in cache_add_dev()  argument
855 	rc = cpu_cache_sysfs_init(cpu);  in cache_add_dev()
859 	parent = per_cpu_cache_dev(cpu);  in cache_add_dev()
860 	for (i = 0; i < cache_leaves(cpu); i++) {  in cache_add_dev()
861 		this_leaf = per_cpu_cacheinfo_idx(cpu, i);  in cache_add_dev()
873 		per_cache_index_dev(cpu, i) = ci_dev;  in cache_add_dev()
875 	cpumask_set_cpu(cpu, &cache_dev_map);  in cache_add_dev()
879 	cpu_cache_sysfs_exit(cpu);  in cache_add_dev()
883 static unsigned int cpu_map_shared_cache(bool online, unsigned int cpu,  in cpu_map_shared_cache()  argument
889 	if (!last_level_cache_is_valid(cpu))  in cpu_map_shared_cache()
892 	llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);  in cpu_map_shared_cache()
902 	/* shared_cpu_map of offlined CPU will be cleared, so use sibling map */  in cpu_map_shared_cache()
904 		if (sibling == cpu || !last_level_cache_is_valid(sibling))  in cpu_map_shared_cache()
915  * Calculate the size of the per-CPU data cache slice.  This can be
917  * by one CPU under ideal circumstances.  UNIFIED caches are counted
922  * available, we just use the size of the per-CPU slice of LLC to make
925 static void update_per_cpu_data_slice_size_cpu(unsigned int cpu)  in update_per_cpu_data_slice_size_cpu()  argument
931 	if (!last_level_cache_is_valid(cpu))  in update_per_cpu_data_slice_size_cpu()
934 	ci = ci_cacheinfo(cpu);  in update_per_cpu_data_slice_size_cpu()
935 	llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);  in update_per_cpu_data_slice_size_cpu()
945 static void update_per_cpu_data_slice_size(bool cpu_online, unsigned int cpu,  in update_per_cpu_data_slice_size()  argument
951 		if (!cpu_online && icpu == cpu)  in update_per_cpu_data_slice_size()
958 static int cacheinfo_cpu_online(unsigned int cpu)  in cacheinfo_cpu_online()  argument
960 	int rc = detect_cache_attributes(cpu);  in cacheinfo_cpu_online()
965 	rc = cache_add_dev(cpu);  in cacheinfo_cpu_online()
968 	if (cpu_map_shared_cache(true, cpu, &cpu_map))  in cacheinfo_cpu_online()
969 		update_per_cpu_data_slice_size(true, cpu, cpu_map);  in cacheinfo_cpu_online()
972 	free_cache_attributes(cpu);  in cacheinfo_cpu_online()
976 static int cacheinfo_cpu_pre_down(unsigned int cpu)  in cacheinfo_cpu_pre_down()  argument
981 	nr_shared = cpu_map_shared_cache(false, cpu, &cpu_map);  in cacheinfo_cpu_pre_down()
982 	if (cpumask_test_and_clear_cpu(cpu, &cache_dev_map))  in cacheinfo_cpu_pre_down()
983 		cpu_cache_sysfs_exit(cpu);  in cacheinfo_cpu_pre_down()
985 	free_cache_attributes(cpu);  in cacheinfo_cpu_pre_down()
987 		update_per_cpu_data_slice_size(false, cpu, cpu_map);  in cacheinfo_cpu_pre_down()