39 static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)  in __enter_from_kernel_mode()  argument
41 	regs->exit_rcu = false;  in __enter_from_kernel_mode()
48 		regs->exit_rcu = true;  in __enter_from_kernel_mode()
57 static void noinstr enter_from_kernel_mode(struct pt_regs *regs)  in enter_from_kernel_mode()  argument
59 	__enter_from_kernel_mode(regs);  in enter_from_kernel_mode()
72 static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)  in __exit_to_kernel_mode()  argument
76 	if (interrupts_enabled(regs)) {  in __exit_to_kernel_mode()
77 		if (regs->exit_rcu) {  in __exit_to_kernel_mode()
87 		if (regs->exit_rcu)  in __exit_to_kernel_mode()
92 static void noinstr exit_to_kernel_mode(struct pt_regs *regs)  in exit_to_kernel_mode()  argument
95 	__exit_to_kernel_mode(regs);  in exit_to_kernel_mode()
112 static __always_inline void enter_from_user_mode(struct pt_regs *regs)  in enter_from_user_mode()  argument
130 static void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags)  in do_notify_resume()  argument
139 			uprobe_notify_resume(regs);  in do_notify_resume()
148 			do_signal(regs);  in do_notify_resume()
151 			resume_user_mode_work(regs);  in do_notify_resume()
161 static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs)  in exit_to_user_mode_prepare()  argument
169 		do_notify_resume(regs, flags);  in exit_to_user_mode_prepare()
176 static __always_inline void exit_to_user_mode(struct pt_regs *regs)  in exit_to_user_mode()  argument
178 	exit_to_user_mode_prepare(regs);  in exit_to_user_mode()
183 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)  in asm_exit_to_user_mode()  argument
185 	exit_to_user_mode(regs);  in asm_exit_to_user_mode()
193 static void noinstr arm64_enter_nmi(struct pt_regs *regs)  in arm64_enter_nmi()  argument
195 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_nmi()
211 static void noinstr arm64_exit_nmi(struct pt_regs *regs)  in arm64_exit_nmi()  argument
213 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_nmi()
233 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)  in arm64_enter_el1_dbg()  argument
235 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_el1_dbg()
248 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)  in arm64_exit_el1_dbg()  argument
250 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_el1_dbg()
304 static void do_interrupt_handler(struct pt_regs *regs,  in do_interrupt_handler()  argument
307 	struct pt_regs *old_regs = set_irq_regs(regs);  in do_interrupt_handler()
310 		call_on_irq_stack(regs, handler);  in do_interrupt_handler()
312 		handler(regs);  in do_interrupt_handler()
320 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,  in __panic_unhandled()  argument
323 	arm64_enter_nmi(regs);  in __panic_unhandled()
331 	__show_regs(regs);  in __panic_unhandled()
336 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs)	\
339 	__panic_unhandled(regs, desc, read_sysreg(esr_el1));				\
369 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)  in cortex_a76_erratum_1463225_debug_handler()  argument
381 	regs->pstate |= PSR_D_BIT;  in cortex_a76_erratum_1463225_debug_handler()
386 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)  in cortex_a76_erratum_1463225_debug_handler()  argument
426 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)  in el1_abort()  argument
430 	enter_from_kernel_mode(regs);  in el1_abort()
431 	local_daif_inherit(regs);  in el1_abort()
432 	do_mem_abort(far, esr, regs);  in el1_abort()
434 	exit_to_kernel_mode(regs);  in el1_abort()
437 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)  in el1_pc()  argument
441 	enter_from_kernel_mode(regs);  in el1_pc()
442 	local_daif_inherit(regs);  in el1_pc()
443 	do_sp_pc_abort(far, esr, regs);  in el1_pc()
445 	exit_to_kernel_mode(regs);  in el1_pc()
448 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)  in el1_undef()  argument
450 	enter_from_kernel_mode(regs);  in el1_undef()
451 	local_daif_inherit(regs);  in el1_undef()
452 	do_el1_undef(regs, esr);  in el1_undef()
454 	exit_to_kernel_mode(regs);  in el1_undef()
457 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)  in el1_bti()  argument
459 	enter_from_kernel_mode(regs);  in el1_bti()
460 	local_daif_inherit(regs);  in el1_bti()
461 	do_el1_bti(regs, esr);  in el1_bti()
463 	exit_to_kernel_mode(regs);  in el1_bti()
466 static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr)  in el1_gcs()  argument
468 	enter_from_kernel_mode(regs);  in el1_gcs()
469 	local_daif_inherit(regs);  in el1_gcs()
470 	do_el1_gcs(regs, esr);  in el1_gcs()
472 	exit_to_kernel_mode(regs);  in el1_gcs()
475 static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr)  in el1_mops()  argument
477 	enter_from_kernel_mode(regs);  in el1_mops()
478 	local_daif_inherit(regs);  in el1_mops()
479 	do_el1_mops(regs, esr);  in el1_mops()
481 	exit_to_kernel_mode(regs);  in el1_mops()
484 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)  in el1_dbg()  argument
488 	arm64_enter_el1_dbg(regs);  in el1_dbg()
489 	if (!cortex_a76_erratum_1463225_debug_handler(regs))  in el1_dbg()
490 		do_debug_exception(far, esr, regs);  in el1_dbg()
491 	arm64_exit_el1_dbg(regs);  in el1_dbg()
494 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)  in el1_fpac()  argument
496 	enter_from_kernel_mode(regs);  in el1_fpac()
497 	local_daif_inherit(regs);  in el1_fpac()
498 	do_el1_fpac(regs, esr);  in el1_fpac()
500 	exit_to_kernel_mode(regs);  in el1_fpac()
503 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)  in el1h_64_sync_handler()  argument
510 		el1_abort(regs, esr);  in el1h_64_sync_handler()
517 		el1_pc(regs, esr);  in el1h_64_sync_handler()
521 		el1_undef(regs, esr);  in el1h_64_sync_handler()
524 		el1_bti(regs, esr);  in el1h_64_sync_handler()
527 		el1_gcs(regs, esr);  in el1h_64_sync_handler()
530 		el1_mops(regs, esr);  in el1h_64_sync_handler()
536 		el1_dbg(regs, esr);  in el1h_64_sync_handler()
539 		el1_fpac(regs, esr);  in el1h_64_sync_handler()
542 		__panic_unhandled(regs, "64-bit el1h sync", esr);  in el1h_64_sync_handler()
546 static __always_inline void __el1_pnmi(struct pt_regs *regs,  in __el1_pnmi()  argument
549 	arm64_enter_nmi(regs);  in __el1_pnmi()
550 	do_interrupt_handler(regs, handler);  in __el1_pnmi()
551 	arm64_exit_nmi(regs);  in __el1_pnmi()
554 static __always_inline void __el1_irq(struct pt_regs *regs,  in __el1_irq()  argument
557 	enter_from_kernel_mode(regs);  in __el1_irq()
560 	do_interrupt_handler(regs, handler);  in __el1_irq()
565 	exit_to_kernel_mode(regs);  in __el1_irq()
567 static void noinstr el1_interrupt(struct pt_regs *regs,  in el1_interrupt()  argument
572 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))  in el1_interrupt()
573 		__el1_pnmi(regs, handler);  in el1_interrupt()
575 		__el1_irq(regs, handler);  in el1_interrupt()
578 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)  in el1h_64_irq_handler()  argument
580 	el1_interrupt(regs, handle_arch_irq);  in el1h_64_irq_handler()
583 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)  in el1h_64_fiq_handler()  argument
585 	el1_interrupt(regs, handle_arch_fiq);  in el1h_64_fiq_handler()
588 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)  in el1h_64_error_handler()  argument
593 	arm64_enter_nmi(regs);  in el1h_64_error_handler()
594 	do_serror(regs, esr);  in el1h_64_error_handler()
595 	arm64_exit_nmi(regs);  in el1h_64_error_handler()
598 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)  in el0_da()  argument
602 	enter_from_user_mode(regs);  in el0_da()
604 	do_mem_abort(far, esr, regs);  in el0_da()
605 	exit_to_user_mode(regs);  in el0_da()
608 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)  in el0_ia()  argument
620 	enter_from_user_mode(regs);  in el0_ia()
622 	do_mem_abort(far, esr, regs);  in el0_ia()
623 	exit_to_user_mode(regs);  in el0_ia()
626 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_acc()  argument
628 	enter_from_user_mode(regs);  in el0_fpsimd_acc()
630 	do_fpsimd_acc(esr, regs);  in el0_fpsimd_acc()
631 	exit_to_user_mode(regs);  in el0_fpsimd_acc()
634 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)  in el0_sve_acc()  argument
636 	enter_from_user_mode(regs);  in el0_sve_acc()
638 	do_sve_acc(esr, regs);  in el0_sve_acc()
639 	exit_to_user_mode(regs);  in el0_sve_acc()
642 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)  in el0_sme_acc()  argument
644 	enter_from_user_mode(regs);  in el0_sme_acc()
646 	do_sme_acc(esr, regs);  in el0_sme_acc()
647 	exit_to_user_mode(regs);  in el0_sme_acc()
650 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_exc()  argument
652 	enter_from_user_mode(regs);  in el0_fpsimd_exc()
654 	do_fpsimd_exc(esr, regs);  in el0_fpsimd_exc()
655 	exit_to_user_mode(regs);  in el0_fpsimd_exc()
658 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)  in el0_sys()  argument
660 	enter_from_user_mode(regs);  in el0_sys()
662 	do_el0_sys(esr, regs);  in el0_sys()
663 	exit_to_user_mode(regs);  in el0_sys()
666 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)  in el0_pc()  argument
670 	if (!is_ttbr0_addr(instruction_pointer(regs)))  in el0_pc()
673 	enter_from_user_mode(regs);  in el0_pc()
675 	do_sp_pc_abort(far, esr, regs);  in el0_pc()
676 	exit_to_user_mode(regs);  in el0_pc()
679 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)  in el0_sp()  argument
681 	enter_from_user_mode(regs);  in el0_sp()
683 	do_sp_pc_abort(regs->sp, esr, regs);  in el0_sp()
684 	exit_to_user_mode(regs);  in el0_sp()
687 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)  in el0_undef()  argument
689 	enter_from_user_mode(regs);  in el0_undef()
691 	do_el0_undef(regs, esr);  in el0_undef()
692 	exit_to_user_mode(regs);  in el0_undef()
695 static void noinstr el0_bti(struct pt_regs *regs)  in el0_bti()  argument
697 	enter_from_user_mode(regs);  in el0_bti()
699 	do_el0_bti(regs);  in el0_bti()
700 	exit_to_user_mode(regs);  in el0_bti()
703 static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)  in el0_mops()  argument
705 	enter_from_user_mode(regs);  in el0_mops()
707 	do_el0_mops(regs, esr);  in el0_mops()
708 	exit_to_user_mode(regs);  in el0_mops()
711 static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr)  in el0_gcs()  argument
713 	enter_from_user_mode(regs);  in el0_gcs()
715 	do_el0_gcs(regs, esr);  in el0_gcs()
716 	exit_to_user_mode(regs);  in el0_gcs()
719 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)  in el0_inv()  argument
721 	enter_from_user_mode(regs);  in el0_inv()
723 	bad_el0_sync(regs, 0, esr);  in el0_inv()
724 	exit_to_user_mode(regs);  in el0_inv()
727 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)  in el0_dbg()  argument
732 	enter_from_user_mode(regs);  in el0_dbg()
733 	do_debug_exception(far, esr, regs);  in el0_dbg()
735 	exit_to_user_mode(regs);  in el0_dbg()
738 static void noinstr el0_svc(struct pt_regs *regs)  in el0_svc()  argument
740 	enter_from_user_mode(regs);  in el0_svc()
744 	do_el0_svc(regs);  in el0_svc()
745 	exit_to_user_mode(regs);  in el0_svc()
748 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)  in el0_fpac()  argument
750 	enter_from_user_mode(regs);  in el0_fpac()
752 	do_el0_fpac(regs, esr);  in el0_fpac()
753 	exit_to_user_mode(regs);  in el0_fpac()
756 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)  in el0t_64_sync_handler()  argument
762 		el0_svc(regs);  in el0t_64_sync_handler()
765 		el0_da(regs, esr);  in el0t_64_sync_handler()
768 		el0_ia(regs, esr);  in el0t_64_sync_handler()
771 		el0_fpsimd_acc(regs, esr);  in el0t_64_sync_handler()
774 		el0_sve_acc(regs, esr);  in el0t_64_sync_handler()
777 		el0_sme_acc(regs, esr);  in el0t_64_sync_handler()
780 		el0_fpsimd_exc(regs, esr);  in el0t_64_sync_handler()
784 		el0_sys(regs, esr);  in el0t_64_sync_handler()
787 		el0_sp(regs, esr);  in el0t_64_sync_handler()
790 		el0_pc(regs, esr);  in el0t_64_sync_handler()
793 		el0_undef(regs, esr);  in el0t_64_sync_handler()
796 		el0_bti(regs);  in el0t_64_sync_handler()
799 		el0_mops(regs, esr);  in el0t_64_sync_handler()
802 		el0_gcs(regs, esr);  in el0t_64_sync_handler()
808 		el0_dbg(regs, esr);  in el0t_64_sync_handler()
811 		el0_fpac(regs, esr);  in el0t_64_sync_handler()
814 		el0_inv(regs, esr);  in el0t_64_sync_handler()
818 static void noinstr el0_interrupt(struct pt_regs *regs,  in el0_interrupt()  argument
821 	enter_from_user_mode(regs);  in el0_interrupt()
825 	if (regs->pc & BIT(55))  in el0_interrupt()
829 	do_interrupt_handler(regs, handler);  in el0_interrupt()
832 	exit_to_user_mode(regs);  in el0_interrupt()
835 static void noinstr __el0_irq_handler_common(struct pt_regs *regs)  in __el0_irq_handler_common()  argument
837 	el0_interrupt(regs, handle_arch_irq);  in __el0_irq_handler_common()
840 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)  in el0t_64_irq_handler()  argument
842 	__el0_irq_handler_common(regs);  in el0t_64_irq_handler()
845 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)  in __el0_fiq_handler_common()  argument
847 	el0_interrupt(regs, handle_arch_fiq);  in __el0_fiq_handler_common()
850 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)  in el0t_64_fiq_handler()  argument
852 	__el0_fiq_handler_common(regs);  in el0t_64_fiq_handler()
855 static void noinstr __el0_error_handler_common(struct pt_regs *regs)  in __el0_error_handler_common()  argument
859 	enter_from_user_mode(regs);  in __el0_error_handler_common()
861 	arm64_enter_nmi(regs);  in __el0_error_handler_common()
862 	do_serror(regs, esr);  in __el0_error_handler_common()
863 	arm64_exit_nmi(regs);  in __el0_error_handler_common()
865 	exit_to_user_mode(regs);  in __el0_error_handler_common()
868 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)  in el0t_64_error_handler()  argument
870 	__el0_error_handler_common(regs);  in el0t_64_error_handler()
874 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)  in el0_cp15()  argument
876 	enter_from_user_mode(regs);  in el0_cp15()
878 	do_el0_cp15(esr, regs);  in el0_cp15()
879 	exit_to_user_mode(regs);  in el0_cp15()
882 static void noinstr el0_svc_compat(struct pt_regs *regs)  in el0_svc_compat()  argument
884 	enter_from_user_mode(regs);  in el0_svc_compat()
887 	do_el0_svc_compat(regs);  in el0_svc_compat()
888 	exit_to_user_mode(regs);  in el0_svc_compat()
891 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)  in el0t_32_sync_handler()  argument
897 		el0_svc_compat(regs);  in el0t_32_sync_handler()
900 		el0_da(regs, esr);  in el0t_32_sync_handler()
903 		el0_ia(regs, esr);  in el0t_32_sync_handler()
906 		el0_fpsimd_acc(regs, esr);  in el0t_32_sync_handler()
909 		el0_fpsimd_exc(regs, esr);  in el0t_32_sync_handler()
912 		el0_pc(regs, esr);  in el0t_32_sync_handler()
918 		el0_undef(regs, esr);  in el0t_32_sync_handler()
922 		el0_cp15(regs, esr);  in el0t_32_sync_handler()
928 		el0_dbg(regs, esr);  in el0t_32_sync_handler()
931 		el0_inv(regs, esr);  in el0t_32_sync_handler()
935 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)  in el0t_32_irq_handler()  argument
937 	__el0_irq_handler_common(regs);  in el0t_32_irq_handler()
940 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)  in el0t_32_fiq_handler()  argument
942 	__el0_fiq_handler_common(regs);  in el0t_32_fiq_handler()
945 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)  in el0t_32_error_handler()  argument
947 	__el0_error_handler_common(regs);  in el0t_32_error_handler()
957 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)  in handle_bad_stack()  argument
962 	arm64_enter_nmi(regs);  in handle_bad_stack()
963 	panic_bad_stack(regs, esr, far);  in handle_bad_stack()
969 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)  in __sdei_handler()  argument
995 	arm64_enter_nmi(regs);  in __sdei_handler()
996 	ret = do_sdei_event(regs, arg);  in __sdei_handler()
997 	arm64_exit_nmi(regs);  in __sdei_handler()