1 // SPDX-License-Identifier: GPL-2.0-or-later
19 #include <asm/text-patching.h>
37 	unsigned long	value;  member
48 	 * the start of the alt_entry, to support the VDSO. This  in calc_addr()
77 static int patch_feature_section_mask(unsigned long value, unsigned long mask,  in patch_feature_section_mask()  argument
80 	u32 *start, *end, *alt_start, *alt_end, *src, *dest;  in patch_feature_section_mask()  local
82 	start = calc_addr(fcur, fcur->start_off);  in patch_feature_section_mask()
83 	end = calc_addr(fcur, fcur->end_off);  in patch_feature_section_mask()
84 	alt_start = calc_addr(fcur, fcur->alt_start_off);  in patch_feature_section_mask()
85 	alt_end = calc_addr(fcur, fcur->alt_end_off);  in patch_feature_section_mask()
87 	if ((alt_end - alt_start) > (end - start))  in patch_feature_section_mask()
90 	if ((value & fcur->mask & mask) == (fcur->value & mask))  in patch_feature_section_mask()
94 	dest = start;  in patch_feature_section_mask()
108 static void do_feature_fixups_mask(unsigned long value, unsigned long mask,  in do_feature_fixups_mask()  argument
117 		if (patch_feature_section_mask(value, mask, fcur)) {  in do_feature_fixups_mask()
119 			printk("Unable to patch feature section at %p - %p" \  in do_feature_fixups_mask()
120 				" with %p - %p\n",  in do_feature_fixups_mask()
121 				calc_addr(fcur, fcur->start_off),  in do_feature_fixups_mask()
122 				calc_addr(fcur, fcur->end_off),  in do_feature_fixups_mask()
123 				calc_addr(fcur, fcur->alt_start_off),  in do_feature_fixups_mask()
124 				calc_addr(fcur, fcur->alt_end_off));  in do_feature_fixups_mask()
129 void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)  in do_feature_fixups()  argument
131 	do_feature_fixups_mask(value, ~0, fixup_start, fixup_end);  in do_feature_fixups()
141 static int do_patch_fixups(long *start, long *end, unsigned int *instrs, int num)  in do_patch_fixups()  argument
145 	for (i = 0; start < end; start++, i++) {  in do_patch_fixups()
147 		unsigned int *dest = (void *)start + *start;  in do_patch_fixups()
162 static int do_patch_entry_fixups(long *start, long *end, unsigned int *instrs,  in do_patch_entry_fixups()  argument
167 	for (i = 0; start < end; start++, i++) {  in do_patch_entry_fixups()
168 		unsigned int *dest = (void *)start + *start;  in do_patch_entry_fixups()
192 	long *start, *end;  in do_stf_entry_barrier_fixups()  local
195 	start = PTRRELOC(&__start___stf_entry_barrier_fixup);  in do_stf_entry_barrier_fixups()
215 	i = do_patch_entry_fixups(start, end, instrs, types & STF_BARRIER_FALLBACK,  in do_stf_entry_barrier_fixups()
218 	printk(KERN_DEBUG "stf-barrier: patched %d entry locations (%s barrier)\n", i,  in do_stf_entry_barrier_fixups()
229 	long *start, *end;  in do_stf_exit_barrier_fixups()  local
232 	start = PTRRELOC(&__start___stf_exit_barrier_fixup);  in do_stf_exit_barrier_fixups()
262 	i = do_patch_fixups(start, end, instrs, ARRAY_SIZE(instrs));  in do_stf_exit_barrier_fixups()
264 	printk(KERN_DEBUG "stf-barrier: patched %d exit locations (%s barrier)\n", i,  in do_stf_exit_barrier_fixups()
289 	 * The call to the fallback entry flush, and the fallback/sync-ori exit  in do_stf_barrier_fixups()
295 	 * The branch to mark interrupt exits non-reentrant is enabled first,  in do_stf_barrier_fixups()
321 	long *start, *end;  in do_uaccess_flush_fixups()  local
324 	start = PTRRELOC(&__start___uaccess_flush_fixup);  in do_uaccess_flush_fixups()
346 	i = do_patch_fixups(start, end, instrs, ARRAY_SIZE(instrs));  in do_uaccess_flush_fixups()
348 	printk(KERN_DEBUG "uaccess-flush: patched %d locations (%s flush)\n", i,  in do_uaccess_flush_fixups()
362 	long *start, *end;  in __do_entry_flush_fixups()  local
391 	 *  - the mflr		(dest)  in __do_entry_flush_fixups()
392 	 *  - the mtlr		(dest + 2)  in __do_entry_flush_fixups()
393 	 *  - the branch	(dest + 1)  in __do_entry_flush_fixups()
401 	 *  - the branch	(dest + 1)  in __do_entry_flush_fixups()
402 	 *  - the mtlr		(dest + 2)  in __do_entry_flush_fixups()
403 	 *  - the mflr		(dest)  in __do_entry_flush_fixups()
406 	 * semi-patched state.  in __do_entry_flush_fixups()
409 	start = PTRRELOC(&__start___entry_flush_fixup);  in __do_entry_flush_fixups()
411 	i = do_patch_entry_fixups(start, end, instrs, types == L1D_FLUSH_FALLBACK,  in __do_entry_flush_fixups()
414 	start = PTRRELOC(&__start___scv_entry_flush_fixup);  in __do_entry_flush_fixups()
416 	i += do_patch_entry_fixups(start, end, instrs, types == L1D_FLUSH_FALLBACK,  in __do_entry_flush_fixups()
419 	printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i,  in __do_entry_flush_fixups()
446 	long *start, *end;  in __do_rfi_flush_fixups()  local
449 	start = PTRRELOC(&__start___rfi_flush_fixup);  in __do_rfi_flush_fixups()
469 	i = do_patch_fixups(start, end, instrs, ARRAY_SIZE(instrs));  in __do_rfi_flush_fixups()
471 	printk(KERN_DEBUG "rfi-flush: patched %d locations (%s flush)\n", i,  in __do_rfi_flush_fixups()
512 	long *start, *end;  in do_barrier_nospec_fixups_range()  local
515 	start = fixup_start;  in do_barrier_nospec_fixups_range()
521 		pr_info("barrier-nospec: using ORI speculation barrier\n");  in do_barrier_nospec_fixups_range()
525 	i = do_patch_fixups(start, end, &instr, 1);  in do_barrier_nospec_fixups_range()
527 	printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i);  in do_barrier_nospec_fixups_range()
535 	void *start, *end;  in do_barrier_nospec_fixups()  local
537 	start = PTRRELOC(&__start___barrier_nospec_fixup);  in do_barrier_nospec_fixups()
540 	do_barrier_nospec_fixups_range(enable, start, end);  in do_barrier_nospec_fixups()
548 	long *start, *end;  in do_barrier_nospec_fixups_range()  local
551 	start = fixup_start;  in do_barrier_nospec_fixups_range()
558 		pr_info("barrier-nospec: using isync; sync as speculation barrier\n");  in do_barrier_nospec_fixups_range()
563 	i = do_patch_fixups(start, end, instr, ARRAY_SIZE(instr));  in do_barrier_nospec_fixups_range()
565 	printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i);  in do_barrier_nospec_fixups_range()
570 	unsigned int *start, *end;  in patch_btb_flush_section()  local
572 	start = (void *)curr + *curr;  in patch_btb_flush_section()
574 	for (; start < end; start++) {  in patch_btb_flush_section()
575 		pr_devel("patching dest %lx\n", (unsigned long)start);  in patch_btb_flush_section()
576 		patch_instruction(start, ppc_inst(PPC_RAW_NOP()));  in patch_btb_flush_section()
582 	long *start, *end;  in do_btb_flush_fixups()  local
584 	start = PTRRELOC(&__start__btb_flush_fixup);  in do_btb_flush_fixups()
587 	for (; start < end; start += 2)  in do_btb_flush_fixups()
588 		patch_btb_flush_section(start);  in do_btb_flush_fixups()
592 void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end)  in do_lwsync_fixups()  argument
594 	long *start, *end;  in do_lwsync_fixups()  local
597 	if (!(value & CPU_FTR_LWSYNC))  in do_lwsync_fixups()
600 	start = fixup_start;  in do_lwsync_fixups()
603 	for (; start < end; start++) {  in do_lwsync_fixups()
604 		dest = (void *)start + *start;  in do_lwsync_fixups()
620 	end = (void *)src + (__end_interrupts - _stext);  in do_final_fixups()
641 	*PTRRELOC(&saved_cpu_features) = spec->cpu_features;  in apply_feature_fixups()
642 	*PTRRELOC(&saved_mmu_features) = spec->mmu_features;  in apply_feature_fixups()
645 	 * Apply the CPU-specific and firmware specific fixups to kernel text  in apply_feature_fixups()
648 	do_feature_fixups(spec->cpu_features,  in apply_feature_fixups()
652 	do_feature_fixups(spec->mmu_features,  in apply_feature_fixups()
656 	do_lwsync_fixups(spec->cpu_features,  in apply_feature_fixups()
671 	saved_mmu_features |= cur_cpu_spec->mmu_features & mask;  in update_mmu_feature_fixups()
673 	do_feature_fixups_mask(cur_cpu_spec->mmu_features, mask,  in update_mmu_feature_fixups()
694 	WARN(saved_cpu_features != cur_cpu_spec->cpu_features,  in check_features()
696 	WARN(saved_mmu_features != cur_cpu_spec->mmu_features,  in check_features()
710 	if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__);
712 static int patch_feature_section(unsigned long value, struct fixup_entry *fcur)  in patch_feature_section()  argument
714 	return patch_feature_section_mask(value, ~0, fcur);  in patch_feature_section()
722 	return (unsigned long)p - (unsigned long)entry;  in calc_offset()
731 	int size = 4 * (end_ftr_fixup_test1 - ftr_fixup_test1);  in test_basic_patching()
733 	fixup.value = fixup.mask = 8;  in test_basic_patching()
741 	/* Check we don't patch if the value matches */  in test_basic_patching()
745 	/* Check we do patch if the value doesn't match */  in test_basic_patching()
763 	int size = 4 * (end_ftr_fixup_test2 - ftr_fixup_test2);  in test_alternative_patching()
765 	fixup.value = fixup.mask = 0xF;  in test_alternative_patching()
774 	/* Check we don't patch if the value matches */  in test_alternative_patching()
778 	/* Check we do patch if the value doesn't match */  in test_alternative_patching()
795 	int size = 4 * (end_ftr_fixup_test3 - ftr_fixup_test3);  in test_alternative_case_too_big()
797 	fixup.value = fixup.mask = 0xC;  in test_alternative_case_too_big()
822 	int size = 4 * (end_ftr_fixup_test4 - ftr_fixup_test4);  in test_alternative_case_too_small()
825 	/* Check a high-bit flag */  in test_alternative_case_too_small()
826 	flag = 1UL << ((sizeof(unsigned long) - 1) * 8);  in test_alternative_case_too_small()
827 	fixup.value = fixup.mask = flag;  in test_alternative_case_too_small()
836 	/* Check we don't patch if the value matches */  in test_alternative_case_too_small()
840 	/* Check we do patch if the value doesn't match */  in test_alternative_case_too_small()
856 	int size = 4 * (end_ftr_fixup_test5 - ftr_fixup_test5);  in test_alternative_case_with_branch()
866 	int size = 4 * (end_ftr_fixup_test6 - ftr_fixup_test6);  in test_alternative_case_with_external_branch()
876 	int size = 4 * (end_ftr_fixup_test7 - ftr_fixup_test7);  in test_alternative_case_with_branch_to_end()
885 	unsigned long size = ftr_fixup_test_FTR_macros_expected -  in test_cpu_macros()
898 	unsigned long size = ftr_fixup_test_FW_FTR_macros_expected -  in test_fw_macros()
913 	unsigned long size = end_lwsync_fixup_test -  in test_lwsync_macros()
917 	if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) {  in test_lwsync_macros()
933 	int size = sizeof(unsigned int) * (end_ftr_fixup_prefix1 - ftr_fixup_prefix1);  in test_prefix_patching()
935 	fixup.value = fixup.mask = 8;  in test_prefix_patching()
955 	int size = sizeof(unsigned int) * (end_ftr_fixup_prefix2 - ftr_fixup_prefix2);  in test_prefix_alt_patching()
957 	fixup.value = fixup.mask = 8;  in test_prefix_alt_patching()
977 	int size = sizeof(unsigned int) * (end_ftr_fixup_prefix3 - ftr_fixup_prefix3);  in test_prefix_word_alt_patching()
979 	fixup.value = fixup.mask = 8;  in test_prefix_word_alt_patching()
1000 	printk(KERN_DEBUG "Running feature fixup self-tests ...\n");  in test_feature_fixups()