From bd9b51a0f1e5469a33e3d01e79827b16a1ae944f Mon Sep 17 00:00:00 2001 From: Dan Aloni Date: Sat, 8 Oct 2022 20:30:46 +0300 Subject: [PATCH] Changes representative of linux-3.10.0-1160.80.1.el7.tar.xz --- Documentation/kernel-parameters.txt | 36 +- Documentation/spec_ctrl.txt | 7 + Makefile | 2 +- arch/x86/Makefile | 5 +- arch/x86/boot/compressed/mem_encrypt.S | 2 + arch/x86/crypto/aes-i586-asm_32.S | 4 +- arch/x86/crypto/aes-x86_64-asm_64.S | 2 +- arch/x86/crypto/aes_ctrby8_avx-x86_64.S | 2 +- arch/x86/crypto/aesni-intel_asm.S | 42 +- arch/x86/crypto/aesni-intel_avx-x86_64.S | 12 +- arch/x86/crypto/blowfish-avx2-asm_64.S | 12 +- arch/x86/crypto/blowfish-x86_64-asm_64.S | 12 +- arch/x86/crypto/camellia-aesni-avx-asm_64.S | 18 +- arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 18 +- arch/x86/crypto/camellia-x86_64-asm_64.S | 12 +- arch/x86/crypto/cast5-avx-x86_64-asm_64.S | 12 +- arch/x86/crypto/cast6-avx-x86_64-asm_64.S | 16 +- arch/x86/crypto/crc32-pclmul_asm.S | 2 +- arch/x86/crypto/crc32c-pcl-intel-asm_64.S | 2 +- arch/x86/crypto/crct10dif-pcl-asm_64.S | 2 +- arch/x86/crypto/ghash-clmulni-intel_asm.S | 8 +- arch/x86/crypto/salsa20-i586-asm_32.S | 6 +- arch/x86/crypto/salsa20-x86_64-asm_64.S | 6 +- arch/x86/crypto/serpent-avx-x86_64-asm_64.S | 16 +- arch/x86/crypto/serpent-avx2-asm_64.S | 16 +- arch/x86/crypto/serpent-sse2-i586-asm_32.S | 6 +- arch/x86/crypto/serpent-sse2-x86_64-asm_64.S | 6 +- .../crypto/sha-mb/sha1_mb_mgr_flush_avx2.S | 6 +- .../crypto/sha-mb/sha1_mb_mgr_submit_avx2.S | 2 +- arch/x86/crypto/sha-mb/sha1_x8_avx2.S | 2 +- arch/x86/crypto/sha1_avx2_x86_64_asm.S | 2 +- arch/x86/crypto/sha1_ni_asm.S | 2 +- arch/x86/crypto/sha1_ssse3_asm.S | 2 +- arch/x86/crypto/sha256-avx-asm.S | 2 +- arch/x86/crypto/sha256-avx2-asm.S | 2 +- .../sha256-mb/sha256_mb_mgr_flush_avx2.S | 6 +- .../sha256-mb/sha256_mb_mgr_submit_avx2.S | 2 +- arch/x86/crypto/sha256-mb/sha256_x8_avx2.S | 2 +- arch/x86/crypto/sha256-ssse3-asm.S | 2 +- arch/x86/crypto/sha256_ni_asm.S | 2 +- arch/x86/crypto/sha512-avx-asm.S | 2 +- arch/x86/crypto/sha512-avx2-asm.S | 2 +- .../sha512-mb/sha512_mb_mgr_flush_avx2.S | 6 +- .../sha512-mb/sha512_mb_mgr_submit_avx2.S | 2 +- arch/x86/crypto/sha512-mb/sha512_x4_avx2.S | 2 +- arch/x86/crypto/sha512-ssse3-asm.S | 2 +- arch/x86/crypto/twofish-avx-x86_64-asm_64.S | 16 +- arch/x86/crypto/twofish-avx2-asm_64.S | 14 +- arch/x86/crypto/twofish-i586-asm_32.S | 4 +- arch/x86/crypto/twofish-x86_64-asm_64-3way.S | 6 +- arch/x86/crypto/twofish-x86_64-asm_64.S | 4 +- arch/x86/ia32/ia32entry.S | 9 +- arch/x86/include/asm/alternative.h | 1 + arch/x86/include/asm/cpufeature.h | 4 +- arch/x86/include/asm/cpufeatures.h | 540 +++++++++--------- arch/x86/include/asm/disabled-features.h | 10 +- arch/x86/include/asm/linkage.h | 14 + arch/x86/include/asm/msr-index.h | 17 + arch/x86/include/asm/nospec-branch.h | 64 ++- arch/x86/include/asm/paravirt.h | 2 +- arch/x86/include/asm/qspinlock_paravirt.h | 4 +- arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/acpi/wakeup_32.S | 6 +- arch/x86/kernel/alternative.c | 132 ++++- arch/x86/kernel/cpu/amd.c | 46 +- arch/x86/kernel/cpu/bugs.c | 257 ++++++++- arch/x86/kernel/cpu/common.c | 108 ++-- arch/x86/kernel/cpu/cpuid-deps.c | 3 + arch/x86/kernel/cpu/scattered.c | 6 +- arch/x86/kernel/entry.S | 15 + arch/x86/kernel/entry_32.S | 8 +- arch/x86/kernel/entry_64.S | 29 +- arch/x86/kernel/head_32.S | 6 +- arch/x86/kernel/kprobes/core.c | 2 +- arch/x86/kernel/kprobes/ftrace.c | 2 +- arch/x86/kernel/mcount_64.S | 8 +- arch/x86/kernel/module.c | 8 +- arch/x86/kernel/paravirt.c | 2 +- arch/x86/kernel/relocate_kernel_32.S | 8 +- arch/x86/kernel/relocate_kernel_64.S | 8 +- arch/x86/kernel/spec_ctrl.c | 52 +- arch/x86/kernel/vmlinux.lds.S | 9 + arch/x86/kernel/vsyscall_emu_64.S | 3 + arch/x86/kernel/x8664_ksyms_64.c | 9 + arch/x86/kvm/cpuid.h | 2 - arch/x86/kvm/emulate.c | 73 ++- arch/x86/kvm/svm.c | 9 + arch/x86/kvm/vmx.c | 9 + arch/x86/lguest/i386_head.S | 6 +- arch/x86/lib/atomic64_386_32.S | 2 +- arch/x86/lib/atomic64_cx8_32.S | 16 +- arch/x86/lib/checksum_32.S | 8 +- arch/x86/lib/clear_page_64.S | 6 +- arch/x86/lib/cmpxchg16b_emu.S | 4 +- arch/x86/lib/cmpxchg8b_emu.S | 4 +- arch/x86/lib/copy_page_64.S | 4 +- arch/x86/lib/copy_user_64.S | 8 +- arch/x86/lib/copy_user_nocache_64.S | 2 +- arch/x86/lib/csum-copy_64.S | 2 +- arch/x86/lib/getuser.S | 14 +- arch/x86/lib/iomap_copy_64.S | 2 +- arch/x86/lib/memcpy_64.S | 12 +- arch/x86/lib/memmove_64.S | 2 +- arch/x86/lib/memset_64.S | 2 +- arch/x86/lib/msr-reg.S | 4 +- arch/x86/lib/putuser.S | 2 +- arch/x86/lib/retpoline.S | 63 ++ arch/x86/lib/rwlock.S | 4 +- arch/x86/lib/rwsem.S | 8 +- arch/x86/lib/thunk_32.S | 2 +- arch/x86/lib/thunk_64.S | 2 +- arch/x86/math-emu/div_Xsig.S | 2 +- arch/x86/math-emu/div_small.S | 2 +- arch/x86/math-emu/mul_Xsig.S | 6 +- arch/x86/math-emu/polynom_Xsig.S | 2 +- arch/x86/math-emu/reg_norm.S | 6 +- arch/x86/math-emu/reg_round.S | 2 +- arch/x86/math-emu/reg_u_add.S | 2 +- arch/x86/math-emu/reg_u_div.S | 2 +- arch/x86/math-emu/reg_u_mul.S | 2 +- arch/x86/math-emu/reg_u_sub.S | 2 +- arch/x86/math-emu/round_Xsig.S | 4 +- arch/x86/math-emu/shr_Xsig.S | 8 +- arch/x86/math-emu/wm_shrx.S | 16 +- arch/x86/mm/mem_encrypt_boot.S | 4 +- arch/x86/net/bpf_jit.S | 26 +- arch/x86/net/trace_bpf_jit.S | 20 +- arch/x86/platform/olpc/xo1-wakeup.S | 6 +- arch/x86/power/hibernate_asm_32.S | 4 +- arch/x86/power/hibernate_asm_64.S | 4 +- arch/x86/um/checksum_32.S | 8 +- arch/x86/um/setjmp_32.S | 2 +- arch/x86/um/setjmp_64.S | 2 +- arch/x86/vdso/Makefile | 10 +- arch/x86/vdso/vdso32/int80.S | 2 +- arch/x86/vdso/vdso32/syscall.S | 2 +- arch/x86/vdso/vdso32/sysenter.S | 2 +- arch/x86/xen/xen-asm.S | 10 +- arch/x86/xen/xen-asm_32.S | 2 +- drivers/base/cpu.c | 8 + drivers/net/ethernet/mellanox/mlx5/core/cmd.c | 29 +- drivers/net/ethernet/mellanox/mlx5/core/eq.c | 27 + drivers/net/ethernet/mellanox/mlx5/core/fw.c | 76 +++ .../net/ethernet/mellanox/mlx5/core/health.c | 29 +- .../net/ethernet/mellanox/mlx5/core/main.c | 57 +- .../ethernet/mellanox/mlx5/core/mlx5_core.h | 18 +- drivers/scsi/lpfc/lpfc_sli.c | 93 ++- fs/nfs/dir.c | 2 +- include/linux/cpu.h | 2 + include/linux/mlx5/device.h | 4 + include/linux/mlx5/driver.h | 3 + include/linux/mlx5/mlx5_ifc.h | 18 +- kernel/posix-cpu-timers.c | 85 +-- net/sched/cls_route.c | 2 +- scripts/kernel.spec | 64 ++- tools/objtool/arch.h | 2 + tools/objtool/arch/x86/decode.c | 7 +- tools/objtool/builtin-check.c | 103 +++- tools/objtool/elf.c | 217 ++++++- tools/objtool/elf.h | 19 +- tools/objtool/special.c | 6 +- 161 files changed, 2256 insertions(+), 819 deletions(-) create mode 100644 arch/x86/kernel/entry.S diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 1f5b56f092be01..9b478eb4145b5e 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -3317,6 +3317,30 @@ bytes respectively. Such letter suffixes can also be entirely omitted. retain_initrd [RAM] Keep initrd memory after extraction + retbleed= [X86] Control mitigation of RETBleed (Arbitrary + Speculative Code Execution with Return Instructions) + vulnerability. + + off - no mitigation + auto - automatically select a migitation + auto,nosmt - automatically select a mitigation, + disabling SMT if necessary for + the full mitigation (only on Zen1 + and older without STIBP). + ibpb - mitigate short speculation windows on + basic block boundaries too. Safe, highest + perf impact. + unret - force enable untrained return thunks, + only effective on AMD f15h-f17h + based systems. + unret,nosmt - like unret, will disable SMT when STIBP + is not available. + + Selecting 'auto' will choose a mitigation method at run + time according to the CPU. + + Not specifying this option is equivalent to retbleed=auto. + rhash_entries= [KNL,NET] Set number of hash buckets for route cache @@ -3504,13 +3528,23 @@ bytes respectively. Such letter suffixes can also be entirely omitted. Specific mitigations can also be selected manually: - retpoline - replace indirect branches + retpoline[,force] - replace indirect branches ibrs - Intel: Indirect Branch Restricted Speculation (kernel) ibrs_always - Intel: Indirect Branch Restricted Speculation (kernel and user space) Not specifying this option is equivalent to spectre_v2=auto. + NOTE: with the advent of RETBleed, IBRS becomes the + defacto speculation control protection mechanism for + Intel CPUs affected by it. Specifying 'retpoline' as + spectre_v2 mitigation will now become equivalent to + spectre_v2=auto for such CPUs. + + adding ',force' to spectre_v2=retpoline will make it + keep the old behavior, which means the system will + remain VULNERABLE to RETBleed. + spec_store_bypass_disable= [HW] Control Speculative Store Bypass (SSB) Disable mitigation (Speculative Store Bypass vulnerability) diff --git a/Documentation/spec_ctrl.txt b/Documentation/spec_ctrl.txt index 33156460e7b191..f9cb263cd76dfc 100644 --- a/Documentation/spec_ctrl.txt +++ b/Documentation/spec_ctrl.txt @@ -79,6 +79,13 @@ retp_enabled 1: Retpolines are enabled in the kernel, which replace all which adds user-to-user and guest-to-guest protection across context switches. + With the advent of RETBleed, IBRS becomes the defacto + speculation control protection mechanism for Intel CPUs + affected by it, and retp_enabled=1 will fallback to + enabling IBRS instead of Retpolines on those CPUs. + +retp_enabled 2: Retpolines are force-enabled on Intel RETBleed affected CPUs. + ibrs_enabled 0: Disabled ibrs_enabled 1: IBRS enabled in kernel mode. diff --git a/Makefile b/Makefile index d51de1c1b3d5eb..a77fdfdc3c9753 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ EXTRAVERSION = NAME = Unicycling Gorilla RHEL_MAJOR = 7 RHEL_MINOR = 9 -RHEL_RELEASE = 1160.76.1 +RHEL_RELEASE = 1160.80.1 # # DRM backport version diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 2a304906b4bed0..17635b2a1dd222 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -159,9 +159,12 @@ KBUILD_AFLAGS += $(mflags-y) # Avoid indirect branches in kernel to deal with Spectre ifdef CONFIG_RETPOLINE - RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register) + RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register -mfunction-return=thunk-extern) + RETPOLINE_VDSO_CFLAGS += $(call cc-option,-mindirect-branch=thunk-inline -mindirect-branch-register) ifneq ($(RETPOLINE_CFLAGS),) KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE + export RETPOLINE_CFLAGS + export RETPOLINE_VDSO_CFLAGS else $(error CONFIG_RETPOLINE=y, but not supported by the compiler. Compiler update recommended.) endif diff --git a/arch/x86/boot/compressed/mem_encrypt.S b/arch/x86/boot/compressed/mem_encrypt.S index 69c86bd81b5e6c..8583662fad8caa 100644 --- a/arch/x86/boot/compressed/mem_encrypt.S +++ b/arch/x86/boot/compressed/mem_encrypt.S @@ -68,6 +68,7 @@ ENTRY(get_sev_encryption_bit) #endif /* CONFIG_AMD_MEM_ENCRYPT */ ret + int3 ENDPROC(get_sev_encryption_bit) .code64 @@ -96,4 +97,5 @@ ENTRY(get_sev_encryption_mask) #endif ret + int3 ENDPROC(get_sev_encryption_mask) diff --git a/arch/x86/crypto/aes-i586-asm_32.S b/arch/x86/crypto/aes-i586-asm_32.S index 2849dbc59e1173..84f9c095d65e78 100644 --- a/arch/x86/crypto/aes-i586-asm_32.S +++ b/arch/x86/crypto/aes-i586-asm_32.S @@ -286,7 +286,7 @@ ENTRY(aes_enc_blk) pop %ebx mov %r0,(%ebp) pop %ebp - ret + RET ENDPROC(aes_enc_blk) // AES (Rijndael) Decryption Subroutine @@ -358,5 +358,5 @@ ENTRY(aes_dec_blk) pop %ebx mov %r0,(%ebp) pop %ebp - ret + RET ENDPROC(aes_dec_blk) diff --git a/arch/x86/crypto/aes-x86_64-asm_64.S b/arch/x86/crypto/aes-x86_64-asm_64.S index 91056554716355..9f0b8a7b6d445e 100644 --- a/arch/x86/crypto/aes-x86_64-asm_64.S +++ b/arch/x86/crypto/aes-x86_64-asm_64.S @@ -77,7 +77,7 @@ movl r6 ## E,4(r9); \ movl r7 ## E,8(r9); \ movl r8 ## E,12(r9); \ - ret; \ + RET; \ ENDPROC(FUNC); #define round(TAB,OFFSET,r1,r2,r3,r4,r5,r6,r7,r8,ra,rb,rc,rd) \ diff --git a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S index a916c4a611652f..d870d3dcc7fd13 100644 --- a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S +++ b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S @@ -537,7 +537,7 @@ ddq_add_8: /* return updated IV */ vpshufb xbyteswap, xcounter, xcounter vmovdqu xcounter, (p_iv) - ret + RET .endm /* diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index 429be3c61c3f94..87267301f9e803 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -1566,7 +1566,7 @@ _return_T_done_decrypt: pop %r14 pop %r13 pop %r12 - ret + RET ENDPROC(aesni_gcm_dec) @@ -1847,7 +1847,7 @@ _return_T_done_encrypt: pop %r14 pop %r13 pop %r12 - ret + RET ENDPROC(aesni_gcm_enc) #endif @@ -1864,7 +1864,7 @@ _key_expansion_256a: pxor %xmm1, %xmm0 movaps %xmm0, (TKEYP) add $0x10, TKEYP - ret + RET ENDPROC(_key_expansion_128) ENDPROC(_key_expansion_256a) @@ -1890,7 +1890,7 @@ _key_expansion_192a: shufps $0b01001110, %xmm2, %xmm1 movaps %xmm1, 0x10(TKEYP) add $0x20, TKEYP - ret + RET ENDPROC(_key_expansion_192a) .align 4 @@ -1910,7 +1910,7 @@ _key_expansion_192b: movaps %xmm0, (TKEYP) add $0x10, TKEYP - ret + RET ENDPROC(_key_expansion_192b) .align 4 @@ -1923,7 +1923,7 @@ _key_expansion_256b: pxor %xmm1, %xmm2 movaps %xmm2, (TKEYP) add $0x10, TKEYP - ret + RET ENDPROC(_key_expansion_256b) /* @@ -2038,7 +2038,7 @@ ENTRY(aesni_set_key) popl KEYP #endif FRAME_END - ret + RET ENDPROC(aesni_set_key) /* @@ -2062,7 +2062,7 @@ ENTRY(aesni_enc) popl KEYP #endif FRAME_END - ret + RET ENDPROC(aesni_enc) /* @@ -2120,7 +2120,7 @@ _aesni_enc1: AESENC KEY STATE movaps 0x70(TKEYP), KEY AESENCLAST KEY STATE - ret + RET ENDPROC(_aesni_enc1) /* @@ -2229,7 +2229,7 @@ _aesni_enc4: AESENCLAST KEY STATE2 AESENCLAST KEY STATE3 AESENCLAST KEY STATE4 - ret + RET ENDPROC(_aesni_enc4) /* @@ -2254,7 +2254,7 @@ ENTRY(aesni_dec) popl KEYP #endif FRAME_END - ret + RET ENDPROC(aesni_dec) /* @@ -2312,7 +2312,7 @@ _aesni_dec1: AESDEC KEY STATE movaps 0x70(TKEYP), KEY AESDECLAST KEY STATE - ret + RET ENDPROC(_aesni_dec1) /* @@ -2421,7 +2421,7 @@ _aesni_dec4: AESDECLAST KEY STATE2 AESDECLAST KEY STATE3 AESDECLAST KEY STATE4 - ret + RET ENDPROC(_aesni_dec4) /* @@ -2481,7 +2481,7 @@ ENTRY(aesni_ecb_enc) popl LEN #endif FRAME_END - ret + RET ENDPROC(aesni_ecb_enc) /* @@ -2542,7 +2542,7 @@ ENTRY(aesni_ecb_dec) popl LEN #endif FRAME_END - ret + RET ENDPROC(aesni_ecb_dec) /* @@ -2586,7 +2586,7 @@ ENTRY(aesni_cbc_enc) popl IVP #endif FRAME_END - ret + RET ENDPROC(aesni_cbc_enc) /* @@ -2679,7 +2679,7 @@ ENTRY(aesni_cbc_dec) popl IVP #endif FRAME_END - ret + RET ENDPROC(aesni_cbc_dec) #ifdef __x86_64__ @@ -2708,7 +2708,7 @@ _aesni_inc_init: mov $1, TCTR_LOW MOVQ_R64_XMM TCTR_LOW INC MOVQ_R64_XMM CTR TCTR_LOW - ret + RET ENDPROC(_aesni_inc_init) /* @@ -2737,7 +2737,7 @@ _aesni_inc: .Linc_low: movaps CTR, IV PSHUFB_XMM BSWAP_MASK IV - ret + RET ENDPROC(_aesni_inc) /* @@ -2800,7 +2800,7 @@ ENTRY(aesni_ctr_enc) movups IV, (IVP) .Lctr_enc_just_ret: FRAME_END - ret + RET ENDPROC(aesni_ctr_enc) /* @@ -2928,7 +2928,7 @@ ENTRY(aesni_xts_crypt8) movdqu STATE4, 0x70(OUTP) FRAME_END - ret + RET ENDPROC(aesni_xts_crypt8) #endif diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S index b9da0daff62b9f..189eeec6c4cfa8 100644 --- a/arch/x86/crypto/aesni-intel_avx-x86_64.S +++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S @@ -1557,7 +1557,7 @@ ENTRY(aesni_gcm_precomp_avx_gen2) pop %r14 pop %r13 pop %r12 - ret + RET ENDPROC(aesni_gcm_precomp_avx_gen2) ############################################################################### @@ -1578,7 +1578,7 @@ ENDPROC(aesni_gcm_precomp_avx_gen2) ############################################################################### ENTRY(aesni_gcm_enc_avx_gen2) GCM_ENC_DEC_AVX ENC - ret + RET ENDPROC(aesni_gcm_enc_avx_gen2) ############################################################################### @@ -1599,7 +1599,7 @@ ENDPROC(aesni_gcm_enc_avx_gen2) ############################################################################### ENTRY(aesni_gcm_dec_avx_gen2) GCM_ENC_DEC_AVX DEC - ret + RET ENDPROC(aesni_gcm_dec_avx_gen2) #endif /* CONFIG_AS_AVX */ @@ -2881,7 +2881,7 @@ ENTRY(aesni_gcm_precomp_avx_gen4) pop %r14 pop %r13 pop %r12 - ret + RET ENDPROC(aesni_gcm_precomp_avx_gen4) @@ -2903,7 +2903,7 @@ ENDPROC(aesni_gcm_precomp_avx_gen4) ############################################################################### ENTRY(aesni_gcm_enc_avx_gen4) GCM_ENC_DEC_AVX2 ENC - ret + RET ENDPROC(aesni_gcm_enc_avx_gen4) ############################################################################### @@ -2924,7 +2924,7 @@ ENDPROC(aesni_gcm_enc_avx_gen4) ############################################################################### ENTRY(aesni_gcm_dec_avx_gen4) GCM_ENC_DEC_AVX2 DEC - ret + RET ENDPROC(aesni_gcm_dec_avx_gen4) #endif /* CONFIG_AS_AVX2 */ diff --git a/arch/x86/crypto/blowfish-avx2-asm_64.S b/arch/x86/crypto/blowfish-avx2-asm_64.S index 784452e0d05d50..aef72d9c3511bf 100644 --- a/arch/x86/crypto/blowfish-avx2-asm_64.S +++ b/arch/x86/crypto/blowfish-avx2-asm_64.S @@ -221,7 +221,7 @@ __blowfish_enc_blk32: write_block(RXl, RXr); - ret; + RET; ENDPROC(__blowfish_enc_blk32) .align 8 @@ -250,7 +250,7 @@ __blowfish_dec_blk32: write_block(RXl, RXr); - ret; + RET; ENDPROC(__blowfish_dec_blk32) ENTRY(blowfish_ecb_enc_32way) @@ -284,7 +284,7 @@ ENTRY(blowfish_ecb_enc_32way) vzeroupper; - ret; + RET; ENDPROC(blowfish_ecb_enc_32way) ENTRY(blowfish_ecb_dec_32way) @@ -318,7 +318,7 @@ ENTRY(blowfish_ecb_dec_32way) vzeroupper; - ret; + RET; ENDPROC(blowfish_ecb_dec_32way) ENTRY(blowfish_cbc_dec_32way) @@ -365,7 +365,7 @@ ENTRY(blowfish_cbc_dec_32way) vzeroupper; - ret; + RET; ENDPROC(blowfish_cbc_dec_32way) ENTRY(blowfish_ctr_32way) @@ -445,5 +445,5 @@ ENTRY(blowfish_ctr_32way) vzeroupper; - ret; + RET; ENDPROC(blowfish_ctr_32way) diff --git a/arch/x86/crypto/blowfish-x86_64-asm_64.S b/arch/x86/crypto/blowfish-x86_64-asm_64.S index 246c67006ed06a..c78de2fa31ef2b 100644 --- a/arch/x86/crypto/blowfish-x86_64-asm_64.S +++ b/arch/x86/crypto/blowfish-x86_64-asm_64.S @@ -149,10 +149,10 @@ ENTRY(__blowfish_enc_blk) jnz .L__enc_xor; write_block(); - ret; + RET; .L__enc_xor: xor_block(); - ret; + RET; ENDPROC(__blowfish_enc_blk) ENTRY(blowfish_dec_blk) @@ -183,7 +183,7 @@ ENTRY(blowfish_dec_blk) movq %r11, %rbp; - ret; + RET; ENDPROC(blowfish_dec_blk) /********************************************************************** @@ -334,14 +334,14 @@ ENTRY(__blowfish_enc_blk_4way) popq %rbx; popq %rbp; - ret; + RET; .L__enc_xor4: xor_block4(); popq %rbx; popq %rbp; - ret; + RET; ENDPROC(__blowfish_enc_blk_4way) ENTRY(blowfish_dec_blk_4way) @@ -375,5 +375,5 @@ ENTRY(blowfish_dec_blk_4way) popq %rbx; popq %rbp; - ret; + RET; ENDPROC(blowfish_dec_blk_4way) diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S index 77ff4de2224d92..19d7de86012557 100644 --- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S +++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S @@ -193,7 +193,7 @@ roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd: roundsm16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, %rcx, (%r9)); - ret; + RET; ENDPROC(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) .align 8 @@ -201,7 +201,7 @@ roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab: roundsm16(%xmm4, %xmm5, %xmm6, %xmm7, %xmm0, %xmm1, %xmm2, %xmm3, %xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11, %rax, (%r9)); - ret; + RET; ENDPROC(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) /* @@ -784,7 +784,7 @@ __camellia_enc_blk16: %xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax)); FRAME_END - ret; + RET; .align 8 .Lenc_max32: @@ -871,7 +871,7 @@ __camellia_dec_blk16: %xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax)); FRAME_END - ret; + RET; .align 8 .Ldec_max32: @@ -912,7 +912,7 @@ ENTRY(camellia_ecb_enc_16way) %xmm8, %rsi); FRAME_END - ret; + RET; ENDPROC(camellia_ecb_enc_16way) ENTRY(camellia_ecb_dec_16way) @@ -942,7 +942,7 @@ ENTRY(camellia_ecb_dec_16way) %xmm8, %rsi); FRAME_END - ret; + RET; ENDPROC(camellia_ecb_dec_16way) ENTRY(camellia_cbc_dec_16way) @@ -993,7 +993,7 @@ ENTRY(camellia_cbc_dec_16way) %xmm8, %rsi); FRAME_END - ret; + RET; ENDPROC(camellia_cbc_dec_16way) #define inc_le128(x, minus_one, tmp) \ @@ -1106,7 +1106,7 @@ ENTRY(camellia_ctr_16way) %xmm8, %rsi); FRAME_END - ret; + RET; ENDPROC(camellia_ctr_16way) #define gf128mul_x_ble(iv, mask, tmp) \ @@ -1250,7 +1250,7 @@ camellia_xts_crypt_16way: %xmm8, %rsi); FRAME_END - ret; + RET; ENDPROC(camellia_xts_crypt_16way) ENTRY(camellia_xts_enc_16way) diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S index 511fc58d840672..690d773b205099 100644 --- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S +++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S @@ -214,7 +214,7 @@ roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd: roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %rcx, (%r9)); - ret; + RET; ENDPROC(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) .align 8 @@ -222,7 +222,7 @@ roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab: roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3, %ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11, %rax, (%r9)); - ret; + RET; ENDPROC(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) /* @@ -806,7 +806,7 @@ __camellia_enc_blk32: %ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax)); FRAME_END - ret; + RET; .align 8 .Lenc_max32: @@ -893,7 +893,7 @@ __camellia_dec_blk32: %ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax)); FRAME_END - ret; + RET; .align 8 .Ldec_max32: @@ -938,7 +938,7 @@ ENTRY(camellia_ecb_enc_32way) vzeroupper; FRAME_END - ret; + RET; ENDPROC(camellia_ecb_enc_32way) ENTRY(camellia_ecb_dec_32way) @@ -972,7 +972,7 @@ ENTRY(camellia_ecb_dec_32way) vzeroupper; FRAME_END - ret; + RET; ENDPROC(camellia_ecb_dec_32way) ENTRY(camellia_cbc_dec_32way) @@ -1040,7 +1040,7 @@ ENTRY(camellia_cbc_dec_32way) vzeroupper; FRAME_END - ret; + RET; ENDPROC(camellia_cbc_dec_32way) #define inc_le128(x, minus_one, tmp) \ @@ -1180,7 +1180,7 @@ ENTRY(camellia_ctr_32way) vzeroupper; FRAME_END - ret; + RET; ENDPROC(camellia_ctr_32way) #define gf128mul_x_ble(iv, mask, tmp) \ @@ -1347,7 +1347,7 @@ camellia_xts_crypt_32way: vzeroupper; FRAME_END - ret; + RET; ENDPROC(camellia_xts_crypt_32way) ENTRY(camellia_xts_enc_32way) diff --git a/arch/x86/crypto/camellia-x86_64-asm_64.S b/arch/x86/crypto/camellia-x86_64-asm_64.S index 310319c601ede2..09782171c9a55f 100644 --- a/arch/x86/crypto/camellia-x86_64-asm_64.S +++ b/arch/x86/crypto/camellia-x86_64-asm_64.S @@ -228,13 +228,13 @@ ENTRY(__camellia_enc_blk) enc_outunpack(mov, RT1); movq RRBP, %rbp; - ret; + RET; .L__enc_xor: enc_outunpack(xor, RT1); movq RRBP, %rbp; - ret; + RET; ENDPROC(__camellia_enc_blk) ENTRY(camellia_dec_blk) @@ -272,7 +272,7 @@ ENTRY(camellia_dec_blk) dec_outunpack(); movq RRBP, %rbp; - ret; + RET; ENDPROC(camellia_dec_blk) /********************************************************************** @@ -463,14 +463,14 @@ ENTRY(__camellia_enc_blk_2way) movq RRBP, %rbp; popq %rbx; - ret; + RET; .L__enc2_xor: enc_outunpack2(xor, RT2); movq RRBP, %rbp; popq %rbx; - ret; + RET; ENDPROC(__camellia_enc_blk_2way) ENTRY(camellia_dec_blk_2way) @@ -510,5 +510,5 @@ ENTRY(camellia_dec_blk_2way) movq RRBP, %rbp; movq RXOR, %rbx; - ret; + RET; ENDPROC(camellia_dec_blk_2way) diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S index 14fa1966bf01d4..92d211eb8683ed 100644 --- a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S @@ -282,7 +282,7 @@ __cast5_enc_blk16: outunpack_blocks(RR3, RL3, RTMP, RX, RKM); outunpack_blocks(RR4, RL4, RTMP, RX, RKM); - ret; + RET; ENDPROC(__cast5_enc_blk16) .align 16 @@ -353,7 +353,7 @@ __cast5_dec_blk16: outunpack_blocks(RR3, RL3, RTMP, RX, RKM); outunpack_blocks(RR4, RL4, RTMP, RX, RKM); - ret; + RET; .L__skip_dec: vpsrldq $4, RKR, RKR; @@ -391,7 +391,7 @@ ENTRY(cast5_ecb_enc_16way) vmovdqu RL4, (7*4*4)(%r11); FRAME_END - ret; + RET; ENDPROC(cast5_ecb_enc_16way) ENTRY(cast5_ecb_dec_16way) @@ -425,7 +425,7 @@ ENTRY(cast5_ecb_dec_16way) vmovdqu RL4, (7*4*4)(%r11); FRAME_END - ret; + RET; ENDPROC(cast5_ecb_dec_16way) ENTRY(cast5_cbc_dec_16way) @@ -476,7 +476,7 @@ ENTRY(cast5_cbc_dec_16way) popq %r12; FRAME_END - ret; + RET; ENDPROC(cast5_cbc_dec_16way) ENTRY(cast5_ctr_16way) @@ -551,5 +551,5 @@ ENTRY(cast5_ctr_16way) popq %r12; FRAME_END - ret; + RET; ENDPROC(cast5_ctr_16way) diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S index c419389889cddf..ceb46b0aa86c8b 100644 --- a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S @@ -296,7 +296,7 @@ __cast6_enc_blk8: outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); - ret; + RET; ENDPROC(__cast6_enc_blk8) .align 8 @@ -341,7 +341,7 @@ __cast6_dec_blk8: outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); - ret; + RET; ENDPROC(__cast6_dec_blk8) ENTRY(cast6_ecb_enc_8way) @@ -361,7 +361,7 @@ ENTRY(cast6_ecb_enc_8way) store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; ENDPROC(cast6_ecb_enc_8way) ENTRY(cast6_ecb_dec_8way) @@ -381,7 +381,7 @@ ENTRY(cast6_ecb_dec_8way) store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; ENDPROC(cast6_ecb_dec_8way) ENTRY(cast6_cbc_dec_8way) @@ -406,7 +406,7 @@ ENTRY(cast6_cbc_dec_8way) popq %r12; FRAME_END - ret; + RET; ENDPROC(cast6_cbc_dec_8way) ENTRY(cast6_ctr_8way) @@ -433,7 +433,7 @@ ENTRY(cast6_ctr_8way) popq %r12; FRAME_END - ret; + RET; ENDPROC(cast6_ctr_8way) ENTRY(cast6_xts_enc_8way) @@ -457,7 +457,7 @@ ENTRY(cast6_xts_enc_8way) store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; ENDPROC(cast6_xts_enc_8way) ENTRY(cast6_xts_dec_8way) @@ -481,5 +481,5 @@ ENTRY(cast6_xts_dec_8way) store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; ENDPROC(cast6_xts_dec_8way) diff --git a/arch/x86/crypto/crc32-pclmul_asm.S b/arch/x86/crypto/crc32-pclmul_asm.S index f247304299a28b..317c6c2f34e938 100644 --- a/arch/x86/crypto/crc32-pclmul_asm.S +++ b/arch/x86/crypto/crc32-pclmul_asm.S @@ -242,5 +242,5 @@ fold_64: pxor %xmm2, %xmm1 PEXTRD 0x01, %xmm1, %eax - ret + RET ENDPROC(crc32_pclmul_le_16) diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S index 358bd59de4d882..5416371bdae381 100644 --- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S +++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S @@ -313,7 +313,7 @@ do_return: popq %rsi popq %rdi popq %rbx - ret + RET ENDPROC(crc_pcl) .section .rodata, "a", %progbits diff --git a/arch/x86/crypto/crct10dif-pcl-asm_64.S b/arch/x86/crypto/crct10dif-pcl-asm_64.S index 35e97569d05ffe..38d154c91649bd 100644 --- a/arch/x86/crypto/crct10dif-pcl-asm_64.S +++ b/arch/x86/crypto/crct10dif-pcl-asm_64.S @@ -367,7 +367,7 @@ _cleanup: # scale the result back to 16 bits shr $16, %eax mov %rcx, %rsp - ret + RET ######################################################################## diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/arch/x86/crypto/ghash-clmulni-intel_asm.S index 88ef500be1f0a7..8eb93a62f6814c 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_asm.S +++ b/arch/x86/crypto/ghash-clmulni-intel_asm.S @@ -94,7 +94,7 @@ __clmul_gf128mul_ble: psrlq $1, T2 pxor T2, T1 pxor T1, DATA - ret + RET ENDPROC(__clmul_gf128mul_ble) /* void clmul_ghash_mul(char *dst, const be128 *shash) */ @@ -108,7 +108,7 @@ ENTRY(clmul_ghash_mul) PSHUFB_XMM BSWAP DATA movups DATA, (%rdi) FRAME_END - ret + RET ENDPROC(clmul_ghash_mul) /* @@ -137,7 +137,7 @@ ENTRY(clmul_ghash_update) movups DATA, (%rdi) .Lupdate_just_ret: FRAME_END - ret + RET ENDPROC(clmul_ghash_update) /* @@ -162,5 +162,5 @@ ENTRY(clmul_ghash_setkey) pand .Lpoly, %xmm1 pxor %xmm1, %xmm0 movups %xmm0, (%rdi) - ret + RET ENDPROC(clmul_ghash_setkey) diff --git a/arch/x86/crypto/salsa20-i586-asm_32.S b/arch/x86/crypto/salsa20-i586-asm_32.S index 329452b8f79466..4b4ecfe5539cff 100644 --- a/arch/x86/crypto/salsa20-i586-asm_32.S +++ b/arch/x86/crypto/salsa20-i586-asm_32.S @@ -924,7 +924,7 @@ ENTRY(salsa20_encrypt_bytes) movl 96(%esp),%ebp # leave add %eax,%esp - ret + RET ._bytesatleast65: # bytes -= 64 sub $64,%ebx @@ -1059,7 +1059,7 @@ ENTRY(salsa20_keysetup) movl 80(%esp),%ebp # leave add %eax,%esp - ret + RET ENDPROC(salsa20_keysetup) # enter salsa20_ivsetup @@ -1110,5 +1110,5 @@ ENTRY(salsa20_ivsetup) movl 80(%esp),%ebp # leave add %eax,%esp - ret + RET ENDPROC(salsa20_ivsetup) diff --git a/arch/x86/crypto/salsa20-x86_64-asm_64.S b/arch/x86/crypto/salsa20-x86_64-asm_64.S index 9279e0b2d60ec8..ce8670df0e8e67 100644 --- a/arch/x86/crypto/salsa20-x86_64-asm_64.S +++ b/arch/x86/crypto/salsa20-x86_64-asm_64.S @@ -789,7 +789,7 @@ ENTRY(salsa20_encrypt_bytes) add %r11,%rsp mov %rdi,%rax mov %rsi,%rdx - ret + RET # bytesatleast65: ._bytesatleast65: # bytes -= 64 @@ -889,7 +889,7 @@ ENTRY(salsa20_keysetup) add %r11,%rsp mov %rdi,%rax mov %rsi,%rdx - ret + RET ENDPROC(salsa20_keysetup) # enter salsa20_ivsetup @@ -914,5 +914,5 @@ ENTRY(salsa20_ivsetup) add %r11,%rsp mov %rdi,%rax mov %rsi,%rdx - ret + RET ENDPROC(salsa20_ivsetup) diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S index 8be571808342b8..87f28bfa8969a4 100644 --- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S @@ -619,7 +619,7 @@ __serpent_enc_blk8_avx: write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; ENDPROC(__serpent_enc_blk8_avx) .align 8 @@ -673,7 +673,7 @@ __serpent_dec_blk8_avx: write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2); write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2); - ret; + RET; ENDPROC(__serpent_dec_blk8_avx) ENTRY(serpent_ecb_enc_8way_avx) @@ -691,7 +691,7 @@ ENTRY(serpent_ecb_enc_8way_avx) store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; ENDPROC(serpent_ecb_enc_8way_avx) ENTRY(serpent_ecb_dec_8way_avx) @@ -709,7 +709,7 @@ ENTRY(serpent_ecb_dec_8way_avx) store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); FRAME_END - ret; + RET; ENDPROC(serpent_ecb_dec_8way_avx) ENTRY(serpent_cbc_dec_8way_avx) @@ -727,7 +727,7 @@ ENTRY(serpent_cbc_dec_8way_avx) store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); FRAME_END - ret; + RET; ENDPROC(serpent_cbc_dec_8way_avx) ENTRY(serpent_ctr_8way_avx) @@ -747,7 +747,7 @@ ENTRY(serpent_ctr_8way_avx) store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; ENDPROC(serpent_ctr_8way_avx) ENTRY(serpent_xts_enc_8way_avx) @@ -769,7 +769,7 @@ ENTRY(serpent_xts_enc_8way_avx) store_xts_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; ENDPROC(serpent_xts_enc_8way_avx) ENTRY(serpent_xts_dec_8way_avx) @@ -791,5 +791,5 @@ ENTRY(serpent_xts_dec_8way_avx) store_xts_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); FRAME_END - ret; + RET; ENDPROC(serpent_xts_dec_8way_avx) diff --git a/arch/x86/crypto/serpent-avx2-asm_64.S b/arch/x86/crypto/serpent-avx2-asm_64.S index 97c48add33ed96..5705dffcd8819b 100644 --- a/arch/x86/crypto/serpent-avx2-asm_64.S +++ b/arch/x86/crypto/serpent-avx2-asm_64.S @@ -611,7 +611,7 @@ __serpent_enc_blk16: write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; ENDPROC(__serpent_enc_blk16) .align 8 @@ -665,7 +665,7 @@ __serpent_dec_blk16: write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2); write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2); - ret; + RET; ENDPROC(__serpent_dec_blk16) ENTRY(serpent_ecb_enc_16way) @@ -687,7 +687,7 @@ ENTRY(serpent_ecb_enc_16way) vzeroupper; FRAME_END - ret; + RET; ENDPROC(serpent_ecb_enc_16way) ENTRY(serpent_ecb_dec_16way) @@ -709,7 +709,7 @@ ENTRY(serpent_ecb_dec_16way) vzeroupper; FRAME_END - ret; + RET; ENDPROC(serpent_ecb_dec_16way) ENTRY(serpent_cbc_dec_16way) @@ -732,7 +732,7 @@ ENTRY(serpent_cbc_dec_16way) vzeroupper; FRAME_END - ret; + RET; ENDPROC(serpent_cbc_dec_16way) ENTRY(serpent_ctr_16way) @@ -757,7 +757,7 @@ ENTRY(serpent_ctr_16way) vzeroupper; FRAME_END - ret; + RET; ENDPROC(serpent_ctr_16way) ENTRY(serpent_xts_enc_16way) @@ -783,7 +783,7 @@ ENTRY(serpent_xts_enc_16way) vzeroupper; FRAME_END - ret; + RET; ENDPROC(serpent_xts_enc_16way) ENTRY(serpent_xts_dec_16way) @@ -809,5 +809,5 @@ ENTRY(serpent_xts_dec_16way) vzeroupper; FRAME_END - ret; + RET; ENDPROC(serpent_xts_dec_16way) diff --git a/arch/x86/crypto/serpent-sse2-i586-asm_32.S b/arch/x86/crypto/serpent-sse2-i586-asm_32.S index d348f1553a793b..458ed6f1338976 100644 --- a/arch/x86/crypto/serpent-sse2-i586-asm_32.S +++ b/arch/x86/crypto/serpent-sse2-i586-asm_32.S @@ -568,12 +568,12 @@ ENTRY(__serpent_enc_blk_4way) write_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE); - ret; + RET; .L__enc_xor4: xor_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE); - ret; + RET; ENDPROC(__serpent_enc_blk_4way) ENTRY(serpent_dec_blk_4way) @@ -627,5 +627,5 @@ ENTRY(serpent_dec_blk_4way) movl arg_dst(%esp), %eax; write_blocks(%eax, RC, RD, RB, RE, RT0, RT1, RA); - ret; + RET; ENDPROC(serpent_dec_blk_4way) diff --git a/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S b/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S index acc066c7c6b21c..456e6013b2d21f 100644 --- a/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S +++ b/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S @@ -690,13 +690,13 @@ ENTRY(__serpent_enc_blk_8way) write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; .L__enc_xor8: xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; ENDPROC(__serpent_enc_blk_8way) ENTRY(serpent_dec_blk_8way) @@ -750,5 +750,5 @@ ENTRY(serpent_dec_blk_8way) write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2); write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2); - ret; + RET; ENDPROC(serpent_dec_blk_8way) diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S index 96df6a39d7e240..97b3b6393f6a7f 100644 --- a/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S +++ b/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S @@ -215,7 +215,7 @@ len_is_0: return: pop %rbx FRAME_END - ret + RET return_null: xor job_rax, job_rax @@ -273,12 +273,12 @@ ENTRY(sha1_mb_mgr_get_comp_job_avx2) pop %rbx - ret + RET .return_null: xor job_rax, job_rax pop %rbx - ret + RET ENDPROC(sha1_mb_mgr_get_comp_job_avx2) .data diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S index 1435acf46f22d3..8c36d16135854b 100644 --- a/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S +++ b/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S @@ -195,7 +195,7 @@ return: pop %r12 pop %rbx FRAME_END - ret + RET return_null: xor job_rax, job_rax diff --git a/arch/x86/crypto/sha-mb/sha1_x8_avx2.S b/arch/x86/crypto/sha-mb/sha1_x8_avx2.S index c9dae1cd291927..d01459809a7f3a 100644 --- a/arch/x86/crypto/sha-mb/sha1_x8_avx2.S +++ b/arch/x86/crypto/sha-mb/sha1_x8_avx2.S @@ -457,7 +457,7 @@ lloop: pop %r13 pop %r12 - ret + RET ENDPROC(sha1_x8_avx2) diff --git a/arch/x86/crypto/sha1_avx2_x86_64_asm.S b/arch/x86/crypto/sha1_avx2_x86_64_asm.S index 1eab79c9ac4841..a07468fdbdf084 100644 --- a/arch/x86/crypto/sha1_avx2_x86_64_asm.S +++ b/arch/x86/crypto/sha1_avx2_x86_64_asm.S @@ -676,7 +676,7 @@ _loop3: pop %rbp pop %rbx - ret + RET ENDPROC(\name) .endm diff --git a/arch/x86/crypto/sha1_ni_asm.S b/arch/x86/crypto/sha1_ni_asm.S index 874a651b9e7dd3..e346e35a44e0d7 100644 --- a/arch/x86/crypto/sha1_ni_asm.S +++ b/arch/x86/crypto/sha1_ni_asm.S @@ -290,7 +290,7 @@ ENTRY(sha1_ni_transform) .Ldone_hash: mov RSPSAVE, %rsp - ret + RET ENDPROC(sha1_ni_transform) .data diff --git a/arch/x86/crypto/sha1_ssse3_asm.S b/arch/x86/crypto/sha1_ssse3_asm.S index a4109506a5e888..eee433409a12c0 100644 --- a/arch/x86/crypto/sha1_ssse3_asm.S +++ b/arch/x86/crypto/sha1_ssse3_asm.S @@ -104,7 +104,7 @@ pop %r12 pop %rbp pop %rbx - ret + RET ENDPROC(\name) .endm diff --git a/arch/x86/crypto/sha256-avx-asm.S b/arch/x86/crypto/sha256-avx-asm.S index 92b3b5d75ba9d8..0dfb39dedc418b 100644 --- a/arch/x86/crypto/sha256-avx-asm.S +++ b/arch/x86/crypto/sha256-avx-asm.S @@ -460,7 +460,7 @@ done_hash: popq %r13 popq %rbp popq %rbx - ret + RET ENDPROC(sha256_transform_avx) .data diff --git a/arch/x86/crypto/sha256-avx2-asm.S b/arch/x86/crypto/sha256-avx2-asm.S index 570ec5ec62d740..b348ed680a546a 100644 --- a/arch/x86/crypto/sha256-avx2-asm.S +++ b/arch/x86/crypto/sha256-avx2-asm.S @@ -720,7 +720,7 @@ done_hash: popq %r12 popq %rbp popq %rbx - ret + RET ENDPROC(sha256_transform_rorx) .data diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S index b691da981cd9cf..fd932813c8b3d7 100644 --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S +++ b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S @@ -215,7 +215,7 @@ len_is_0: return: pop %rbx FRAME_END - ret + RET return_null: xor job_rax, job_rax @@ -275,12 +275,12 @@ ENTRY(sha256_mb_mgr_get_comp_job_avx2) pop %rbx - ret + RET .return_null: xor job_rax, job_rax pop %rbx - ret + RET ENDPROC(sha256_mb_mgr_get_comp_job_avx2) .data diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S index 7ea670e25acc82..a973c91118fd57 100644 --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S +++ b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S @@ -200,7 +200,7 @@ return: pop %r12 pop %rbx FRAME_END - ret + RET return_null: xor job_rax, job_rax diff --git a/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S b/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S index aa21aea4c722d0..ff6fb2fcba918b 100644 --- a/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S +++ b/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S @@ -435,7 +435,7 @@ Lrounds_16_xx: pop %r13 pop %r12 - ret + RET ENDPROC(sha256_x8_avx2) .data .align 64 diff --git a/arch/x86/crypto/sha256-ssse3-asm.S b/arch/x86/crypto/sha256-ssse3-asm.S index 2cedc44e81216a..bf5fccdcbe508a 100644 --- a/arch/x86/crypto/sha256-ssse3-asm.S +++ b/arch/x86/crypto/sha256-ssse3-asm.S @@ -471,7 +471,7 @@ done_hash: popq %rbp popq %rbx - ret + RET ENDPROC(sha256_transform_ssse3) .data diff --git a/arch/x86/crypto/sha256_ni_asm.S b/arch/x86/crypto/sha256_ni_asm.S index 748cdf21a938ba..32fff04e4624ba 100644 --- a/arch/x86/crypto/sha256_ni_asm.S +++ b/arch/x86/crypto/sha256_ni_asm.S @@ -326,7 +326,7 @@ ENTRY(sha256_ni_transform) .Ldone_hash: - ret + RET ENDPROC(sha256_ni_transform) .data diff --git a/arch/x86/crypto/sha512-avx-asm.S b/arch/x86/crypto/sha512-avx-asm.S index 565274d6a6419f..41ef727c1538dd 100644 --- a/arch/x86/crypto/sha512-avx-asm.S +++ b/arch/x86/crypto/sha512-avx-asm.S @@ -364,7 +364,7 @@ updateblock: mov frame_RSPSAVE(%rsp), %rsp nowork: - ret + RET ENDPROC(sha512_transform_avx) ######################################################################## diff --git a/arch/x86/crypto/sha512-avx2-asm.S b/arch/x86/crypto/sha512-avx2-asm.S index 1f20b35d8573a1..b743c98e3a4ce6 100644 --- a/arch/x86/crypto/sha512-avx2-asm.S +++ b/arch/x86/crypto/sha512-avx2-asm.S @@ -678,7 +678,7 @@ done_hash: # Restore Stack Pointer mov frame_RSPSAVE(%rsp), %rsp - ret + RET ENDPROC(sha512_transform_rorx) ######################################################################## diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S index 3ddba19a0db61c..bf7225eb5273b9 100644 --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S +++ b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S @@ -212,7 +212,7 @@ len_is_0: return: pop %rbx FRAME_END - ret + RET return_null: xor job_rax, job_rax @@ -273,12 +273,12 @@ ENTRY(sha512_mb_mgr_get_comp_job_avx2) pop %rbx - ret + RET .return_null: xor job_rax, job_rax pop %rbx - ret + RET ENDPROC(sha512_mb_mgr_get_comp_job_avx2) .data diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S index 815f07bdd1f8f4..9d2cab28810433 100644 --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S +++ b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S @@ -203,7 +203,7 @@ return: pop %r12 pop %rbx FRAME_END - ret + RET return_null: xor job_rax, job_rax diff --git a/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S b/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S index 31ab1eff64133c..8c0ee0d6ec7ae8 100644 --- a/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S +++ b/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S @@ -358,7 +358,7 @@ Lrounds_16_xx: pop %r12 # outer calling routine restores XMM and other GP registers - ret + RET ENDPROC(sha512_x4_avx2) .data diff --git a/arch/x86/crypto/sha512-ssse3-asm.S b/arch/x86/crypto/sha512-ssse3-asm.S index e610e29cbc8184..00bb34354f38c4 100644 --- a/arch/x86/crypto/sha512-ssse3-asm.S +++ b/arch/x86/crypto/sha512-ssse3-asm.S @@ -363,7 +363,7 @@ updateblock: mov frame_RSPSAVE(%rsp), %rsp nowork: - ret + RET ENDPROC(sha512_transform_ssse3) ######################################################################## diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S index dc66273e610d97..c915c9b900d7e5 100644 --- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S @@ -285,7 +285,7 @@ __twofish_enc_blk8: outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); - ret; + RET; ENDPROC(__twofish_enc_blk8) .align 8 @@ -325,7 +325,7 @@ __twofish_dec_blk8: outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2); outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2); - ret; + RET; ENDPROC(__twofish_dec_blk8) ENTRY(twofish_ecb_enc_8way) @@ -345,7 +345,7 @@ ENTRY(twofish_ecb_enc_8way) store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); FRAME_END - ret; + RET; ENDPROC(twofish_ecb_enc_8way) ENTRY(twofish_ecb_dec_8way) @@ -365,7 +365,7 @@ ENTRY(twofish_ecb_dec_8way) store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; ENDPROC(twofish_ecb_dec_8way) ENTRY(twofish_cbc_dec_8way) @@ -390,7 +390,7 @@ ENTRY(twofish_cbc_dec_8way) popq %r12; FRAME_END - ret; + RET; ENDPROC(twofish_cbc_dec_8way) ENTRY(twofish_ctr_8way) @@ -417,7 +417,7 @@ ENTRY(twofish_ctr_8way) popq %r12; FRAME_END - ret; + RET; ENDPROC(twofish_ctr_8way) ENTRY(twofish_xts_enc_8way) @@ -441,7 +441,7 @@ ENTRY(twofish_xts_enc_8way) store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); FRAME_END - ret; + RET; ENDPROC(twofish_xts_enc_8way) ENTRY(twofish_xts_dec_8way) @@ -465,5 +465,5 @@ ENTRY(twofish_xts_dec_8way) store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; ENDPROC(twofish_xts_dec_8way) diff --git a/arch/x86/crypto/twofish-avx2-asm_64.S b/arch/x86/crypto/twofish-avx2-asm_64.S index e1a83b9cd389b7..55101ecf56d287 100644 --- a/arch/x86/crypto/twofish-avx2-asm_64.S +++ b/arch/x86/crypto/twofish-avx2-asm_64.S @@ -422,7 +422,7 @@ __twofish_enc_blk16: outunpack_enc16(RA, RB, RC, RD); write_blocks16(RA, RB, RC, RD); - ret; + RET; ENDPROC(__twofish_enc_blk16) .align 8 @@ -454,7 +454,7 @@ __twofish_dec_blk16: outunpack_dec16(RA, RB, RC, RD); write_blocks16(RA, RB, RC, RD); - ret; + RET; ENDPROC(__twofish_dec_blk16) ENTRY(twofish_ecb_enc_16way) @@ -476,7 +476,7 @@ ENTRY(twofish_ecb_enc_16way) popq %r12; vzeroupper; - ret; + RET; ENDPROC(twofish_ecb_enc_16way) ENTRY(twofish_ecb_dec_16way) @@ -498,7 +498,7 @@ ENTRY(twofish_ecb_dec_16way) popq %r12; vzeroupper; - ret; + RET; ENDPROC(twofish_ecb_dec_16way) ENTRY(twofish_cbc_dec_16way) @@ -521,7 +521,7 @@ ENTRY(twofish_cbc_dec_16way) popq %r12; vzeroupper; - ret; + RET; ENDPROC(twofish_cbc_dec_16way) ENTRY(twofish_ctr_16way) @@ -546,7 +546,7 @@ ENTRY(twofish_ctr_16way) popq %r12; vzeroupper; - ret; + RET; ENDPROC(twofish_ctr_16way) .align 8 @@ -574,7 +574,7 @@ twofish_xts_crypt_16way: popq %r12; vzeroupper; - ret; + RET; ENDPROC(twofish_xts_crypt_16way) ENTRY(twofish_xts_enc_16way) diff --git a/arch/x86/crypto/twofish-i586-asm_32.S b/arch/x86/crypto/twofish-i586-asm_32.S index 694ea4587ba7df..b19440cbb7b8db 100644 --- a/arch/x86/crypto/twofish-i586-asm_32.S +++ b/arch/x86/crypto/twofish-i586-asm_32.S @@ -273,7 +273,7 @@ ENTRY(twofish_enc_blk) pop %ebx pop %ebp mov $1, %eax - ret + RET ENDPROC(twofish_enc_blk) ENTRY(twofish_dec_blk) @@ -330,5 +330,5 @@ ENTRY(twofish_dec_blk) pop %ebx pop %ebp mov $1, %eax - ret + RET ENDPROC(twofish_dec_blk) diff --git a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S index 1c3b7ceb36d24c..cfd64814f0d037 100644 --- a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S +++ b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S @@ -258,7 +258,7 @@ ENTRY(__twofish_enc_blk_3way) popq %r13; popq %r14; popq %r15; - ret; + RET; .L__enc_xor3: outunpack_enc3(xor); @@ -269,7 +269,7 @@ ENTRY(__twofish_enc_blk_3way) popq %r13; popq %r14; popq %r15; - ret; + RET; ENDPROC(__twofish_enc_blk_3way) ENTRY(twofish_dec_blk_3way) @@ -308,5 +308,5 @@ ENTRY(twofish_dec_blk_3way) popq %r13; popq %r14; popq %r15; - ret; + RET; ENDPROC(twofish_dec_blk_3way) diff --git a/arch/x86/crypto/twofish-x86_64-asm_64.S b/arch/x86/crypto/twofish-x86_64-asm_64.S index a039d21986a21c..62e6881a1aac5b 100644 --- a/arch/x86/crypto/twofish-x86_64-asm_64.S +++ b/arch/x86/crypto/twofish-x86_64-asm_64.S @@ -265,7 +265,7 @@ ENTRY(twofish_enc_blk) popq R1 movq $1,%rax - ret + RET ENDPROC(twofish_enc_blk) ENTRY(twofish_dec_blk) @@ -317,5 +317,5 @@ ENTRY(twofish_dec_blk) popq R1 movq $1,%rax - ret + RET ENDPROC(twofish_dec_blk) diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S index d19bf1a7c9e2ab..8403e6f7f1ed90 100644 --- a/arch/x86/ia32/ia32entry.S +++ b/arch/x86/ia32/ia32entry.S @@ -129,7 +129,8 @@ ENTRY(ia32_sysenter_target) addq $(KERNEL_STACK_OFFSET),%rsp IBRS_ENTRY CLEAR_R8_TO_R15 - FILL_RETURN_BUFFER /* no ret allowed before stuffing the RSB */ + FILL_RETURN_BUFFER %rax, X86_FEATURE_RSB_CTXSW /* no ret allowed before stuffing the RSB */ + UNTRAIN_RET /* * No need to follow this irqs on/off section: the syscall * disabled irqs, here we enable it straight after entry: @@ -303,7 +304,8 @@ ENTRY(ia32_cstar_target) CFI_REGISTER rsp,r8 movq PER_CPU_VAR(kernel_stack),%rsp IBRS_ENTRY - FILL_RETURN_BUFFER /* no ret allowed before stuffing the RSB */ + FILL_RETURN_BUFFER %rax, X86_FEATURE_RSB_CTXSW /* no ret allowed before stuffing the RSB */ + UNTRAIN_RET /* * No need to follow this irqs on/off section: the syscall * disabled irqs and here we enable it straight after entry: @@ -479,7 +481,8 @@ ENTRY(ia32_syscall) movq %r9, %rcx IBRS_ENTRY - FILL_RETURN_BUFFER /* no ret allowed before stuffing the RSB */ + FILL_RETURN_BUFFER %rax, X86_FEATURE_RSB_CTXSW /* no ret allowed before stuffing the RSB */ + UNTRAIN_RET /* * No need to follow this irqs on/off section: the syscall diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index 20d74d2c35dabf..5deda34ca18239 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -55,6 +55,7 @@ struct alt_instr { extern void alternative_instructions(void); extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); +extern void apply_returns(s32 *start, s32 *end); struct module; diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index ffc30cdd9c759b..828aaf8827f7d7 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h @@ -24,8 +24,8 @@ enum cpuid_leafs CPUID_LNX_3, CPUID_7_0_EBX, CPUID_D_1_EAX, - CPUID_F_0_EDX, - CPUID_F_1_EDX, + CPUID_LNX_4, + CPUID_DUMMY, CPUID_8000_0008_EBX, CPUID_6_EAX, CPUID_8000_000A_EDX, diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 7cf672fd2f5685..b7d733af58f473 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -12,177 +12,175 @@ /* * Defines x86 CPU feature bits */ -#define NCAPINTS 20 /* N 32-bit words worth of info */ -#define NBUGINTS 10 /* N 32-bit bug flags */ +#define NCAPINTS 20 /* N 32-bit words worth of info */ +#define NBUGINTS 10 /* N 32-bit bug flags */ /* * Note: If the comment begins with a quoted string, that string is used * in /proc/cpuinfo instead of the macro name. If the string is "", * this feature bit is not displayed in /proc/cpuinfo at all. - */ - -/* + * * When adding new features here that depend on other features, - * please update the table in kernel/cpu/cpuid-deps.c + * please update the table in kernel/cpu/cpuid-deps.c as well. */ -/* Intel-defined CPU features, CPUID level 0x00000001 (edx), word 0 */ -#define X86_FEATURE_FPU (0*32+ 0) /* Onboard FPU */ -#define X86_FEATURE_VME (0*32+ 1) /* Virtual Mode Extensions */ -#define X86_FEATURE_DE (0*32+ 2) /* Debugging Extensions */ -#define X86_FEATURE_PSE (0*32+ 3) /* Page Size Extensions */ -#define X86_FEATURE_TSC (0*32+ 4) /* Time Stamp Counter */ -#define X86_FEATURE_MSR (0*32+ 5) /* Model-Specific Registers */ -#define X86_FEATURE_PAE (0*32+ 6) /* Physical Address Extensions */ -#define X86_FEATURE_MCE (0*32+ 7) /* Machine Check Exception */ -#define X86_FEATURE_CX8 (0*32+ 8) /* CMPXCHG8 instruction */ -#define X86_FEATURE_APIC (0*32+ 9) /* Onboard APIC */ -#define X86_FEATURE_SEP (0*32+11) /* SYSENTER/SYSEXIT */ -#define X86_FEATURE_MTRR (0*32+12) /* Memory Type Range Registers */ -#define X86_FEATURE_PGE (0*32+13) /* Page Global Enable */ -#define X86_FEATURE_MCA (0*32+14) /* Machine Check Architecture */ -#define X86_FEATURE_CMOV (0*32+15) /* CMOV instructions */ - /* (plus FCMOVcc, FCOMI with FPU) */ -#define X86_FEATURE_PAT (0*32+16) /* Page Attribute Table */ -#define X86_FEATURE_PSE36 (0*32+17) /* 36-bit PSEs */ -#define X86_FEATURE_PN (0*32+18) /* Processor serial number */ -#define X86_FEATURE_CLFLUSH (0*32+19) /* CLFLUSH instruction */ -#define X86_FEATURE_DS (0*32+21) /* "dts" Debug Store */ -#define X86_FEATURE_ACPI (0*32+22) /* ACPI via MSR */ -#define X86_FEATURE_MMX (0*32+23) /* Multimedia Extensions */ -#define X86_FEATURE_FXSR (0*32+24) /* FXSAVE/FXRSTOR, CR4.OSFXSR */ -#define X86_FEATURE_XMM (0*32+25) /* "sse" */ -#define X86_FEATURE_XMM2 (0*32+26) /* "sse2" */ -#define X86_FEATURE_SELFSNOOP (0*32+27) /* "ss" CPU self snoop */ -#define X86_FEATURE_HT (0*32+28) /* Hyper-Threading */ -#define X86_FEATURE_ACC (0*32+29) /* "tm" Automatic clock control */ -#define X86_FEATURE_IA64 (0*32+30) /* IA-64 processor */ -#define X86_FEATURE_PBE (0*32+31) /* Pending Break Enable */ +/* Intel-defined CPU features, CPUID level 0x00000001 (EDX), word 0 */ +#define X86_FEATURE_FPU ( 0*32+ 0) /* Onboard FPU */ +#define X86_FEATURE_VME ( 0*32+ 1) /* Virtual Mode Extensions */ +#define X86_FEATURE_DE ( 0*32+ 2) /* Debugging Extensions */ +#define X86_FEATURE_PSE ( 0*32+ 3) /* Page Size Extensions */ +#define X86_FEATURE_TSC ( 0*32+ 4) /* Time Stamp Counter */ +#define X86_FEATURE_MSR ( 0*32+ 5) /* Model-Specific Registers */ +#define X86_FEATURE_PAE ( 0*32+ 6) /* Physical Address Extensions */ +#define X86_FEATURE_MCE ( 0*32+ 7) /* Machine Check Exception */ +#define X86_FEATURE_CX8 ( 0*32+ 8) /* CMPXCHG8 instruction */ +#define X86_FEATURE_APIC ( 0*32+ 9) /* Onboard APIC */ +#define X86_FEATURE_SEP ( 0*32+11) /* SYSENTER/SYSEXIT */ +#define X86_FEATURE_MTRR ( 0*32+12) /* Memory Type Range Registers */ +#define X86_FEATURE_PGE ( 0*32+13) /* Page Global Enable */ +#define X86_FEATURE_MCA ( 0*32+14) /* Machine Check Architecture */ +#define X86_FEATURE_CMOV ( 0*32+15) /* CMOV instructions (plus FCMOVcc, FCOMI with FPU) */ +#define X86_FEATURE_PAT ( 0*32+16) /* Page Attribute Table */ +#define X86_FEATURE_PSE36 ( 0*32+17) /* 36-bit PSEs */ +#define X86_FEATURE_PN ( 0*32+18) /* Processor serial number */ +#define X86_FEATURE_CLFLUSH ( 0*32+19) /* CLFLUSH instruction */ +#define X86_FEATURE_DS ( 0*32+21) /* "dts" Debug Store */ +#define X86_FEATURE_ACPI ( 0*32+22) /* ACPI via MSR */ +#define X86_FEATURE_MMX ( 0*32+23) /* Multimedia Extensions */ +#define X86_FEATURE_FXSR ( 0*32+24) /* FXSAVE/FXRSTOR, CR4.OSFXSR */ +#define X86_FEATURE_XMM ( 0*32+25) /* "sse" */ +#define X86_FEATURE_XMM2 ( 0*32+26) /* "sse2" */ +#define X86_FEATURE_SELFSNOOP ( 0*32+27) /* "ss" CPU self snoop */ +#define X86_FEATURE_HT ( 0*32+28) /* Hyper-Threading */ +#define X86_FEATURE_ACC ( 0*32+29) /* "tm" Automatic clock control */ +#define X86_FEATURE_IA64 ( 0*32+30) /* IA-64 processor */ +#define X86_FEATURE_PBE ( 0*32+31) /* Pending Break Enable */ /* AMD-defined CPU features, CPUID level 0x80000001, word 1 */ /* Don't duplicate feature flags which are redundant with Intel! */ -#define X86_FEATURE_SYSCALL (1*32+11) /* SYSCALL/SYSRET */ -#define X86_FEATURE_MP (1*32+19) /* MP Capable. */ -#define X86_FEATURE_NX (1*32+20) /* Execute Disable */ -#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */ -#define X86_FEATURE_FXSR_OPT (1*32+25) /* FXSAVE/FXRSTOR optimizations */ -#define X86_FEATURE_GBPAGES (1*32+26) /* "pdpe1gb" GB pages */ -#define X86_FEATURE_RDTSCP (1*32+27) /* RDTSCP */ -#define X86_FEATURE_LM (1*32+29) /* Long Mode (x86-64) */ -#define X86_FEATURE_3DNOWEXT (1*32+30) /* AMD 3DNow! extensions */ -#define X86_FEATURE_3DNOW (1*32+31) /* 3DNow! */ +#define X86_FEATURE_SYSCALL ( 1*32+11) /* SYSCALL/SYSRET */ +#define X86_FEATURE_MP ( 1*32+19) /* MP Capable */ +#define X86_FEATURE_NX ( 1*32+20) /* Execute Disable */ +#define X86_FEATURE_MMXEXT ( 1*32+22) /* AMD MMX extensions */ +#define X86_FEATURE_FXSR_OPT ( 1*32+25) /* FXSAVE/FXRSTOR optimizations */ +#define X86_FEATURE_GBPAGES ( 1*32+26) /* "pdpe1gb" GB pages */ +#define X86_FEATURE_RDTSCP ( 1*32+27) /* RDTSCP */ +#define X86_FEATURE_LM ( 1*32+29) /* Long Mode (x86-64, 64-bit support) */ +#define X86_FEATURE_3DNOWEXT ( 1*32+30) /* AMD 3DNow extensions */ +#define X86_FEATURE_3DNOW ( 1*32+31) /* 3DNow */ /* Transmeta-defined CPU features, CPUID level 0x80860001, word 2 */ -#define X86_FEATURE_RECOVERY (2*32+ 0) /* CPU in recovery mode */ -#define X86_FEATURE_LONGRUN (2*32+ 1) /* Longrun power control */ -#define X86_FEATURE_LRTI (2*32+ 3) /* LongRun table interface */ +#define X86_FEATURE_RECOVERY ( 2*32+ 0) /* CPU in recovery mode */ +#define X86_FEATURE_LONGRUN ( 2*32+ 1) /* Longrun power control */ +#define X86_FEATURE_LRTI ( 2*32+ 3) /* LongRun table interface */ /* Other features, Linux-defined mapping, word 3 */ /* This range is used for feature bits which conflict or are synthesized */ -#define X86_FEATURE_CXMMX (3*32+ 0) /* Cyrix MMX extensions */ -#define X86_FEATURE_K6_MTRR (3*32+ 1) /* AMD K6 nonstandard MTRRs */ -#define X86_FEATURE_CYRIX_ARR (3*32+ 2) /* Cyrix ARRs (= MTRRs) */ -#define X86_FEATURE_CENTAUR_MCR (3*32+ 3) /* Centaur MCRs (= MTRRs) */ -/* cpu types for specific tunings: */ -#define X86_FEATURE_K8 (3*32+ 4) /* "" Opteron, Athlon64 */ -#define X86_FEATURE_K7 (3*32+ 5) /* "" Athlon */ -#define X86_FEATURE_P3 (3*32+ 6) /* "" P3 */ -#define X86_FEATURE_P4 (3*32+ 7) /* "" P4 */ -#define X86_FEATURE_CONSTANT_TSC (3*32+ 8) /* TSC ticks at a constant rate */ -#define X86_FEATURE_UP (3*32+ 9) /* smp kernel running on up */ -#define X86_FEATURE_FXSAVE_LEAK (3*32+10) /* "" FXSAVE leaks FOP/FIP/FOP */ -#define X86_FEATURE_ART (3*32+10) /* Platform has always running timer (ART) */ -#define X86_FEATURE_ARCH_PERFMON (3*32+11) /* Intel Architectural PerfMon */ -#define X86_FEATURE_PEBS (3*32+12) /* Precise-Event Based Sampling */ -#define X86_FEATURE_BTS (3*32+13) /* Branch Trace Store */ -#define X86_FEATURE_SYSCALL32 (3*32+14) /* "" syscall in ia32 userspace */ -#define X86_FEATURE_SYSENTER32 (3*32+15) /* "" sysenter in ia32 userspace */ -#define X86_FEATURE_REP_GOOD (3*32+16) /* rep microcode works well */ -#define X86_FEATURE_LFENCE_RDTSC (3*32+18) /* "" Lfence synchronizes RDTSC */ -#define X86_FEATURE_11AP (3*32+19) /* "" Bad local APIC aka 11AP */ -#define X86_FEATURE_NOPL (3*32+20) /* The NOPL (0F 1F) instructions */ +#define X86_FEATURE_CXMMX ( 3*32+ 0) /* Cyrix MMX extensions */ +#define X86_FEATURE_K6_MTRR ( 3*32+ 1) /* AMD K6 nonstandard MTRRs */ +#define X86_FEATURE_CYRIX_ARR ( 3*32+ 2) /* Cyrix ARRs (= MTRRs) */ +#define X86_FEATURE_CENTAUR_MCR ( 3*32+ 3) /* Centaur MCRs (= MTRRs) */ + +/* CPU types for specific tunings: */ +#define X86_FEATURE_K8 ( 3*32+ 4) /* "" Opteron, Athlon64 */ +#define X86_FEATURE_K7 ( 3*32+ 5) /* "" Athlon */ +#define X86_FEATURE_P3 ( 3*32+ 6) /* "" P3 */ +#define X86_FEATURE_P4 ( 3*32+ 7) /* "" P4 */ +#define X86_FEATURE_CONSTANT_TSC ( 3*32+ 8) /* TSC ticks at a constant rate */ +#define X86_FEATURE_UP ( 3*32+ 9) /* SMP kernel running on UP */ +#define X86_FEATURE_FXSAVE_LEAK ( 3*32+10) /* "" FXSAVE leaks FOP/FIP/FOP */ +#define X86_FEATURE_ART ( 3*32+10) /* Always running timer (ART) */ +#define X86_FEATURE_ARCH_PERFMON ( 3*32+11) /* Intel Architectural PerfMon */ +#define X86_FEATURE_PEBS ( 3*32+12) /* Precise-Event Based Sampling */ +#define X86_FEATURE_BTS ( 3*32+13) /* Branch Trace Store */ +#define X86_FEATURE_SYSCALL32 ( 3*32+14) /* "" syscall in IA32 userspace */ +#define X86_FEATURE_SYSENTER32 ( 3*32+15) /* "" sysenter in IA32 userspace */ +#define X86_FEATURE_REP_GOOD ( 3*32+16) /* REP microcode works well */ +#define X86_FEATURE_LFENCE_RDTSC ( 3*32+18) /* "" LFENCE synchronizes RDTSC */ +#define X86_FEATURE_11AP ( 3*32+19) /* "" Bad local APIC aka 11AP */ +#define X86_FEATURE_NOPL ( 3*32+20) /* The NOPL (0F 1F) instructions */ /* 21 available, was AMD_C1E */ -#define X86_FEATURE_XTOPOLOGY (3*32+22) /* cpu topology enum extensions */ -#define X86_FEATURE_TSC_RELIABLE (3*32+23) /* TSC is known to be reliable */ -#define X86_FEATURE_NONSTOP_TSC (3*32+24) /* TSC does not stop in C states */ -#define X86_FEATURE_CLFLUSH_MONITOR (3*32+25) /* "" clflush reqd with monitor */ -#define X86_FEATURE_EXTD_APICID (3*32+26) /* has extended APICID (8 bits) */ -#define X86_FEATURE_AMD_DCM (3*32+27) /* multi-node processor */ -#define X86_FEATURE_APERFMPERF (3*32+28) /* APERFMPERF */ -#define X86_FEATURE_EAGER_FPU (3*32+29) /* "eagerfpu" Non lazy FPU restore */ -#define X86_FEATURE_NONSTOP_TSC_S3 (3*32+30) /* TSC doesn't stop in S3 state */ +#define X86_FEATURE_XTOPOLOGY ( 3*32+22) /* CPU topology enum extensions */ +#define X86_FEATURE_TSC_RELIABLE ( 3*32+23) /* TSC is known to be reliable */ +#define X86_FEATURE_NONSTOP_TSC ( 3*32+24) /* TSC does not stop in C states */ +#define X86_FEATURE_CLFLUSH_MONITOR ( 3*32+25) /* "" clflush reqd with monitor */ +#define X86_FEATURE_EXTD_APICID ( 3*32+26) /* Extended APICID (8 bits) */ +#define X86_FEATURE_AMD_DCM ( 3*32+27) /* AMD multi-node processor */ +#define X86_FEATURE_APERFMPERF ( 3*32+28) /* P-State hardware coordination feedback capability (APERF/MPERF MSRs) */ +#define X86_FEATURE_EAGER_FPU ( 3*32+29) /* "eagerfpu" Non lazy FPU restore */ +#define X86_FEATURE_NONSTOP_TSC_S3 ( 3*32+30) /* TSC doesn't stop in S3 state */ -/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ -#define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */ -#define X86_FEATURE_PCLMULQDQ (4*32+ 1) /* PCLMULQDQ instruction */ -#define X86_FEATURE_DTES64 (4*32+ 2) /* 64-bit Debug Store */ -#define X86_FEATURE_MWAIT (4*32+ 3) /* "monitor" Monitor/Mwait support */ -#define X86_FEATURE_DSCPL (4*32+ 4) /* "ds_cpl" CPL Qual. Debug Store */ -#define X86_FEATURE_VMX (4*32+ 5) /* Hardware virtualization */ -#define X86_FEATURE_SMX (4*32+ 6) /* Safer mode */ -#define X86_FEATURE_EST (4*32+ 7) /* Enhanced SpeedStep */ -#define X86_FEATURE_TM2 (4*32+ 8) /* Thermal Monitor 2 */ -#define X86_FEATURE_SSSE3 (4*32+ 9) /* Supplemental SSE-3 */ -#define X86_FEATURE_CID (4*32+10) /* Context ID */ -#define X86_FEATURE_SDBG (4*32+11) /* Silicon Debug */ -#define X86_FEATURE_FMA (4*32+12) /* Fused multiply-add */ -#define X86_FEATURE_CX16 (4*32+13) /* CMPXCHG16B */ -#define X86_FEATURE_XTPR (4*32+14) /* Send Task Priority Messages */ -#define X86_FEATURE_PDCM (4*32+15) /* Performance Capabilities */ -#define X86_FEATURE_PCID (4*32+17) /* Process Context Identifiers */ -#define X86_FEATURE_DCA (4*32+18) /* Direct Cache Access */ -#define X86_FEATURE_XMM4_1 (4*32+19) /* "sse4_1" SSE-4.1 */ -#define X86_FEATURE_XMM4_2 (4*32+20) /* "sse4_2" SSE-4.2 */ -#define X86_FEATURE_X2APIC (4*32+21) /* x2APIC */ -#define X86_FEATURE_MOVBE (4*32+22) /* MOVBE instruction */ -#define X86_FEATURE_POPCNT (4*32+23) /* POPCNT instruction */ -#define X86_FEATURE_TSC_DEADLINE_TIMER (4*32+24) /* Tsc deadline timer */ -#define X86_FEATURE_AES (4*32+25) /* AES instructions */ -#define X86_FEATURE_XSAVE (4*32+26) /* XSAVE/XRSTOR/XSETBV/XGETBV */ -#define X86_FEATURE_OSXSAVE (4*32+27) /* "" XSAVE enabled in the OS */ -#define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */ -#define X86_FEATURE_F16C (4*32+29) /* 16-bit fp conversions */ -#define X86_FEATURE_RDRAND (4*32+30) /* The RDRAND instruction */ -#define X86_FEATURE_HYPERVISOR (4*32+31) /* Running on a hypervisor */ +/* Intel-defined CPU features, CPUID level 0x00000001 (ECX), word 4 */ +#define X86_FEATURE_XMM3 ( 4*32+ 0) /* "pni" SSE-3 */ +#define X86_FEATURE_PCLMULQDQ ( 4*32+ 1) /* PCLMULQDQ instruction */ +#define X86_FEATURE_DTES64 ( 4*32+ 2) /* 64-bit Debug Store */ +#define X86_FEATURE_MWAIT ( 4*32+ 3) /* "monitor" MONITOR/MWAIT support */ +#define X86_FEATURE_DSCPL ( 4*32+ 4) /* "ds_cpl" CPL-qualified (filtered) Debug Store */ +#define X86_FEATURE_VMX ( 4*32+ 5) /* Hardware virtualization */ +#define X86_FEATURE_SMX ( 4*32+ 6) /* Safer Mode eXtensions */ +#define X86_FEATURE_EST ( 4*32+ 7) /* Enhanced SpeedStep */ +#define X86_FEATURE_TM2 ( 4*32+ 8) /* Thermal Monitor 2 */ +#define X86_FEATURE_SSSE3 ( 4*32+ 9) /* Supplemental SSE-3 */ +#define X86_FEATURE_CID ( 4*32+10) /* Context ID */ +#define X86_FEATURE_SDBG ( 4*32+11) /* Silicon Debug */ +#define X86_FEATURE_FMA ( 4*32+12) /* Fused multiply-add */ +#define X86_FEATURE_CX16 ( 4*32+13) /* CMPXCHG16B instruction */ +#define X86_FEATURE_XTPR ( 4*32+14) /* Send Task Priority Messages */ +#define X86_FEATURE_PDCM ( 4*32+15) /* Perf/Debug Capabilities MSR */ +#define X86_FEATURE_PCID ( 4*32+17) /* Process Context Identifiers */ +#define X86_FEATURE_DCA ( 4*32+18) /* Direct Cache Access */ +#define X86_FEATURE_XMM4_1 ( 4*32+19) /* "sse4_1" SSE-4.1 */ +#define X86_FEATURE_XMM4_2 ( 4*32+20) /* "sse4_2" SSE-4.2 */ +#define X86_FEATURE_X2APIC ( 4*32+21) /* X2APIC */ +#define X86_FEATURE_MOVBE ( 4*32+22) /* MOVBE instruction */ +#define X86_FEATURE_POPCNT ( 4*32+23) /* POPCNT instruction */ +#define X86_FEATURE_TSC_DEADLINE_TIMER ( 4*32+24) /* TSC deadline timer */ +#define X86_FEATURE_AES ( 4*32+25) /* AES instructions */ +#define X86_FEATURE_XSAVE ( 4*32+26) /* XSAVE/XRSTOR/XSETBV/XGETBV instructions */ +#define X86_FEATURE_OSXSAVE ( 4*32+27) /* "" XSAVE instruction enabled in the OS */ +#define X86_FEATURE_AVX ( 4*32+28) /* Advanced Vector Extensions */ +#define X86_FEATURE_F16C ( 4*32+29) /* 16-bit FP conversions */ +#define X86_FEATURE_RDRAND ( 4*32+30) /* RDRAND instruction */ +#define X86_FEATURE_HYPERVISOR ( 4*32+31) /* Running on a hypervisor */ /* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */ -#define X86_FEATURE_XSTORE (5*32+ 2) /* "rng" RNG present (xstore) */ -#define X86_FEATURE_XSTORE_EN (5*32+ 3) /* "rng_en" RNG enabled */ -#define X86_FEATURE_XCRYPT (5*32+ 6) /* "ace" on-CPU crypto (xcrypt) */ -#define X86_FEATURE_XCRYPT_EN (5*32+ 7) /* "ace_en" on-CPU crypto enabled */ -#define X86_FEATURE_ACE2 (5*32+ 8) /* Advanced Cryptography Engine v2 */ -#define X86_FEATURE_ACE2_EN (5*32+ 9) /* ACE v2 enabled */ -#define X86_FEATURE_PHE (5*32+10) /* PadLock Hash Engine */ -#define X86_FEATURE_PHE_EN (5*32+11) /* PHE enabled */ -#define X86_FEATURE_PMM (5*32+12) /* PadLock Montgomery Multiplier */ -#define X86_FEATURE_PMM_EN (5*32+13) /* PMM enabled */ +#define X86_FEATURE_XSTORE ( 5*32+ 2) /* "rng" RNG present (xstore) */ +#define X86_FEATURE_XSTORE_EN ( 5*32+ 3) /* "rng_en" RNG enabled */ +#define X86_FEATURE_XCRYPT ( 5*32+ 6) /* "ace" on-CPU crypto (xcrypt) */ +#define X86_FEATURE_XCRYPT_EN ( 5*32+ 7) /* "ace_en" on-CPU crypto enabled */ +#define X86_FEATURE_ACE2 ( 5*32+ 8) /* Advanced Cryptography Engine v2 */ +#define X86_FEATURE_ACE2_EN ( 5*32+ 9) /* ACE v2 enabled */ +#define X86_FEATURE_PHE ( 5*32+10) /* PadLock Hash Engine */ +#define X86_FEATURE_PHE_EN ( 5*32+11) /* PHE enabled */ +#define X86_FEATURE_PMM ( 5*32+12) /* PadLock Montgomery Multiplier */ +#define X86_FEATURE_PMM_EN ( 5*32+13) /* PMM enabled */ -/* More extended AMD flags: CPUID level 0x80000001, ecx, word 6 */ -#define X86_FEATURE_LAHF_LM (6*32+ 0) /* LAHF/SAHF in long mode */ -#define X86_FEATURE_CMP_LEGACY (6*32+ 1) /* If yes HyperThreading not valid */ -#define X86_FEATURE_SVM (6*32+ 2) /* Secure virtual machine */ -#define X86_FEATURE_EXTAPIC (6*32+ 3) /* Extended APIC space */ -#define X86_FEATURE_CR8_LEGACY (6*32+ 4) /* CR8 in 32-bit mode */ -#define X86_FEATURE_ABM (6*32+ 5) /* Advanced bit manipulation */ -#define X86_FEATURE_SSE4A (6*32+ 6) /* SSE-4A */ -#define X86_FEATURE_MISALIGNSSE (6*32+ 7) /* Misaligned SSE mode */ -#define X86_FEATURE_3DNOWPREFETCH (6*32+ 8) /* 3DNow prefetch instructions */ -#define X86_FEATURE_OSVW (6*32+ 9) /* OS Visible Workaround */ -#define X86_FEATURE_IBS (6*32+10) /* Instruction Based Sampling */ -#define X86_FEATURE_XOP (6*32+11) /* extended AVX instructions */ -#define X86_FEATURE_SKINIT (6*32+12) /* SKINIT/STGI instructions */ -#define X86_FEATURE_WDT (6*32+13) /* Watchdog timer */ -#define X86_FEATURE_LWP (6*32+15) /* Light Weight Profiling */ -#define X86_FEATURE_FMA4 (6*32+16) /* 4 operands MAC instructions */ -#define X86_FEATURE_TCE (6*32+17) /* translation cache extension */ -#define X86_FEATURE_NODEID_MSR (6*32+19) /* NodeId MSR */ -#define X86_FEATURE_TBM (6*32+21) /* trailing bit manipulations */ -#define X86_FEATURE_TOPOEXT (6*32+22) /* topology extensions CPUID leafs */ -#define X86_FEATURE_PERFCTR_CORE (6*32+23) /* core performance counter extensions */ -#define X86_FEATURE_PERFCTR_NB (6*32+24) /* NB performance counter extensions */ -#define X86_FEATURE_BPEXT (6*32+26) /* data breakpoint extension */ -#define X86_FEATURE_PTSC (6*32+27) /* performance time-stamp counter */ -#define X86_FEATURE_PERFCTR_L2 (6*32+28) /* L2 performance counter extensions */ +/* More extended AMD flags: CPUID level 0x80000001, ECX, word 6 */ +#define X86_FEATURE_LAHF_LM ( 6*32+ 0) /* LAHF/SAHF in long mode */ +#define X86_FEATURE_CMP_LEGACY ( 6*32+ 1) /* If yes HyperThreading not valid */ +#define X86_FEATURE_SVM ( 6*32+ 2) /* Secure Virtual Machine */ +#define X86_FEATURE_EXTAPIC ( 6*32+ 3) /* Extended APIC space */ +#define X86_FEATURE_CR8_LEGACY ( 6*32+ 4) /* CR8 in 32-bit mode */ +#define X86_FEATURE_ABM ( 6*32+ 5) /* Advanced bit manipulation */ +#define X86_FEATURE_SSE4A ( 6*32+ 6) /* SSE-4A */ +#define X86_FEATURE_MISALIGNSSE ( 6*32+ 7) /* Misaligned SSE mode */ +#define X86_FEATURE_3DNOWPREFETCH ( 6*32+ 8) /* 3DNow prefetch instructions */ +#define X86_FEATURE_OSVW ( 6*32+ 9) /* OS Visible Workaround */ +#define X86_FEATURE_IBS ( 6*32+10) /* Instruction Based Sampling */ +#define X86_FEATURE_XOP ( 6*32+11) /* extended AVX instructions */ +#define X86_FEATURE_SKINIT ( 6*32+12) /* SKINIT/STGI instructions */ +#define X86_FEATURE_WDT ( 6*32+13) /* Watchdog timer */ +#define X86_FEATURE_LWP ( 6*32+15) /* Light Weight Profiling */ +#define X86_FEATURE_FMA4 ( 6*32+16) /* 4 operands MAC instructions */ +#define X86_FEATURE_TCE ( 6*32+17) /* Translation Cache Extension */ +#define X86_FEATURE_NODEID_MSR ( 6*32+19) /* NodeId MSR */ +#define X86_FEATURE_TBM ( 6*32+21) /* Trailing Bit Manipulations */ +#define X86_FEATURE_TOPOEXT ( 6*32+22) /* Topology extensions CPUID leafs */ +#define X86_FEATURE_PERFCTR_CORE ( 6*32+23) /* Core performance counter extensions */ +#define X86_FEATURE_PERFCTR_NB ( 6*32+24) /* NB performance counter extensions */ +#define X86_FEATURE_BPEXT ( 6*32+26) /* Data breakpoint extension */ +#define X86_FEATURE_PTSC ( 6*32+27) /* Performance time-stamp counter */ +#define X86_FEATURE_PERFCTR_L2 ( 6*32+28) /* L2 performance counter extensions */ /* * Auxiliary flags: Linux defined - For features scattered in various @@ -191,29 +189,29 @@ * Reuse free bits when adding new feature flags! */ -#define X86_FEATURE_RING3MWAIT (7*32+ 0) /* Ring 3 MONITOR/MWAIT */ -#define X86_FEATURE_FENCE_SWAPGS_USER (7*32+ 1) /* "" LFENCE in user entry SWAPGS path */ -#define X86_FEATURE_CPB (7*32+ 2) /* AMD Core Performance Boost */ -#define X86_FEATURE_EPB (7*32+ 3) /* IA32_ENERGY_PERF_BIAS support */ -#define X86_FEATURE_CAT_L3 (7*32+ 4) /* Cache Allocation Technology L3 */ -#define X86_FEATURE_CAT_L2 (7*32+ 5) /* Cache Allocation Technology L2 */ -#define X86_FEATURE_CDP_L3 (7*32+ 6) /* Code and Data Prioritization L3 */ -#define X86_FEATURE_INVPCID_SINGLE ( 7*32+ 7) /* Effectively INVPCID && CR4.PCIDE=1 */ -#define X86_FEATURE_HW_PSTATE (7*32+ 8) /* AMD HW-PState */ -#define X86_FEATURE_PROC_FEEDBACK (7*32+ 9) /* AMD ProcFeedbackInterface */ -#define X86_FEATURE_SME ( 7*32+10) /* AMD Secure Memory Encryption */ -#define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+11) /* "" MSR SPEC_CTRL is implemented */ -#define X86_FEATURE_FENCE_SWAPGS_KERNEL (7*32+12) /* "" LFENCE in kernel entry SWAPGS path */ -#define X86_FEATURE_RETPOLINE_AMD (7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */ -#define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */ -#define X86_FEATURE_INTEL_PT ( 7*32+15) /* Intel Processor Trace */ -#define X86_FEATURE_CDP_L2 ( 7*32+16) /* Code and Data Prioritization L2 */ -#define X86_FEATURE_SSBD ( 7*32+17) /* Speculative Store Bypass Disable */ -#define X86_FEATURE_MBA ( 7*32+18) /* Memory Bandwidth Allocation */ -#define X86_FEATURE_IBP_DISABLE ( 7*32+21) /* Old AMD Indirect Branch Predictor Disable */ -#define X86_FEATURE_USE_IBPB ( 7*32+22) /* "" Indirect Branch Prediction Barrier enabled */ -#define X86_FEATURE_SPEC_STORE_BYPASS_DISABLE (7*32+23) /* "" Disable Speculative Store Bypass. */ -#define X86_FEATURE_LS_CFG_SSBD ( 7*32+24) /* "" AMD SSBD implementation */ +#define X86_FEATURE_RING3MWAIT ( 7*32+ 0) /* Ring 3 MONITOR/MWAIT instructions */ +#define X86_FEATURE_CPB ( 7*32+ 2) /* AMD Core Performance Boost */ +#define X86_FEATURE_EPB ( 7*32+ 3) /* IA32_ENERGY_PERF_BIAS support */ +#define X86_FEATURE_CAT_L3 ( 7*32+ 4) /* Cache Allocation Technology L3 */ +#define X86_FEATURE_CAT_L2 ( 7*32+ 5) /* Cache Allocation Technology L2 */ +#define X86_FEATURE_CDP_L3 ( 7*32+ 6) /* Code and Data Prioritization L3 */ +#define X86_FEATURE_INVPCID_SINGLE ( 7*32+ 7) /* Effectively INVPCID && CR4.PCIDE=1 */ +#define X86_FEATURE_HW_PSTATE ( 7*32+ 8) /* AMD HW-PState */ +#define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */ +#define X86_FEATURE_SME ( 7*32+10) /* AMD Secure Memory Encryption */ +/* FREE! ( 7*32+12) */ +/* FREE! ( 7*32+13) */ +#define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */ +#define X86_FEATURE_CDP_L2 ( 7*32+15) /* Code and Data Prioritization L2 */ +#define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */ +#define X86_FEATURE_SSBD ( 7*32+17) /* Speculative Store Bypass Disable */ +#define X86_FEATURE_MBA ( 7*32+18) /* Memory Bandwidth Allocation */ +#define X86_FEATURE_RSB_CTXSW ( 7*32+19) /* Fill RSB on context switches */ +/* FREE! ( 7*32+20) */ +#define X86_FEATURE_IBP_DISABLE ( 7*32+21) /* Old AMD Indirect Branch Predictor Disable */ +#define X86_FEATURE_USE_IBPB ( 7*32+22) /* "" Indirect Branch Prediction Barrier enabled */ +#define X86_FEATURE_SPEC_STORE_BYPASS_DISABLE ( 7*32+23) /* "" Disable Speculative Store Bypass. */ +#define X86_FEATURE_LS_CFG_SSBD ( 7*32+24) /* "" AMD SSBD implementation */ #define X86_FEATURE_IBRS ( 7*32+25) /* Indirect Branch Restricted Speculation */ #define X86_FEATURE_IBPB ( 7*32+26) /* Indirect Branch Prediction Barrier */ #define X86_FEATURE_STIBP ( 7*32+27) /* Single Thread Indirect Branch Predictors */ @@ -222,115 +220,128 @@ #define X86_FEATURE_IBRS_ENHANCED ( 7*32+30) /* Enhanced IBRS */ /* Virtualization flags: Linux defined, word 8 */ -#define X86_FEATURE_TPR_SHADOW (8*32+ 0) /* Intel TPR Shadow */ -#define X86_FEATURE_VNMI (8*32+ 1) /* Intel Virtual NMI */ -#define X86_FEATURE_FLEXPRIORITY (8*32+ 2) /* Intel FlexPriority */ -#define X86_FEATURE_EPT (8*32+ 3) /* Intel Extended Page Table */ -#define X86_FEATURE_VPID (8*32+ 4) /* Intel Virtual Processor ID */ -#define X86_FEATURE_VMMCALL (8*32+15) /* Prefer vmmcall to vmcall */ - -/* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */ -#define X86_FEATURE_FSGSBASE (9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/ -#define X86_FEATURE_TSC_ADJUST (9*32+ 1) /* TSC adjustment MSR 0x3b */ -#define X86_FEATURE_BMI1 (9*32+ 3) /* 1st group bit manipulation extensions */ -#define X86_FEATURE_HLE (9*32+ 4) /* Hardware Lock Elision */ -#define X86_FEATURE_AVX2 (9*32+ 5) /* AVX2 instructions */ -#define X86_FEATURE_SMEP (9*32+ 7) /* Supervisor Mode Execution Protection */ -#define X86_FEATURE_BMI2 (9*32+ 8) /* 2nd group bit manipulation extensions */ -#define X86_FEATURE_ERMS (9*32+ 9) /* Enhanced REP MOVSB/STOSB */ -#define X86_FEATURE_INVPCID (9*32+10) /* Invalidate Processor Context ID */ -#define X86_FEATURE_RTM (9*32+11) /* Restricted Transactional Memory */ -#define X86_FEATURE_CQM (9*32+12) /* Cache QoS Monitoring */ -#define X86_FEATURE_MPX (9*32+14) /* Memory Protection Extension */ -#define X86_FEATURE_RDT_A (9*32+15) /* Resource Director Technology Allocation */ -#define X86_FEATURE_AVX512F (9*32+16) /* AVX-512 Foundation */ -#define X86_FEATURE_AVX512DQ ( 9*32+17) /* AVX-512 DQ (Double/Quad granular) Instructions */ -#define X86_FEATURE_RDSEED (9*32+18) /* The RDSEED instruction */ -#define X86_FEATURE_ADX (9*32+19) /* The ADCX and ADOX instructions */ -#define X86_FEATURE_SMAP (9*32+20) /* Supervisor Mode Access Prevention */ -#define X86_FEATURE_AVX512IFMA ( 9*32+21) /* AVX-512 Integer Fused Multiply-Add instructions */ -#define X86_FEATURE_CLFLUSHOPT (9*32+23) /* CLFLUSHOPT instruction */ -#define X86_FEATURE_CLWB ( 9*32+24) /* CLWB instruction */ -#define X86_FEATURE_AVX512PF (9*32+26) /* AVX-512 Prefetch */ -#define X86_FEATURE_AVX512ER (9*32+27) /* AVX-512 Exponential and Reciprocal */ -#define X86_FEATURE_AVX512CD (9*32+28) /* AVX-512 Conflict Detection */ -#define X86_FEATURE_SHA_NI ( 9*32+29) /* SHA1/SHA256 Instruction Extensions */ -#define X86_FEATURE_AVX512BW ( 9*32+30) /* AVX-512 BW (Byte/Word granular) Instructions */ -#define X86_FEATURE_AVX512VL ( 9*32+31) /* AVX-512 VL (128/256 Vector Length) Extensions */ +#define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */ +#define X86_FEATURE_VNMI ( 8*32+ 1) /* Intel Virtual NMI */ +#define X86_FEATURE_FLEXPRIORITY ( 8*32+ 2) /* Intel FlexPriority */ +#define X86_FEATURE_EPT ( 8*32+ 3) /* Intel Extended Page Table */ +#define X86_FEATURE_VPID ( 8*32+ 4) /* Intel Virtual Processor ID */ +#define X86_FEATURE_VMMCALL ( 8*32+15) /* Prefer VMMCALL to VMCALL */ -/* Extended state features, CPUID level 0x0000000d:1 (eax), word 10 */ -#define X86_FEATURE_XSAVEOPT (10*32+ 0) /* XSAVEOPT */ -#define X86_FEATURE_XSAVEC (10*32+ 1) /* XSAVEC */ -#define X86_FEATURE_XGETBV1 (10*32+ 2) /* XGETBV with ECX = 1 */ -#define X86_FEATURE_XSAVES (10*32+ 3) /* XSAVES/XRSTORS */ +/* Intel-defined CPU features, CPUID level 0x00000007:0 (EBX), word 9 */ +#define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* RDFSBASE, WRFSBASE, RDGSBASE, WRGSBASE instructions*/ +#define X86_FEATURE_TSC_ADJUST ( 9*32+ 1) /* TSC adjustment MSR 0x3B */ +#define X86_FEATURE_BMI1 ( 9*32+ 3) /* 1st group bit manipulation extensions */ +#define X86_FEATURE_HLE ( 9*32+ 4) /* Hardware Lock Elision */ +#define X86_FEATURE_AVX2 ( 9*32+ 5) /* AVX2 instructions */ +#define X86_FEATURE_SMEP ( 9*32+ 7) /* Supervisor Mode Execution Protection */ +#define X86_FEATURE_BMI2 ( 9*32+ 8) /* 2nd group bit manipulation extensions */ +#define X86_FEATURE_ERMS ( 9*32+ 9) /* Enhanced REP MOVSB/STOSB instructions */ +#define X86_FEATURE_INVPCID ( 9*32+10) /* Invalidate Processor Context ID */ +#define X86_FEATURE_RTM ( 9*32+11) /* Restricted Transactional Memory */ +#define X86_FEATURE_CQM ( 9*32+12) /* Cache QoS Monitoring */ +#define X86_FEATURE_MPX ( 9*32+14) /* Memory Protection Extension */ +#define X86_FEATURE_RDT_A ( 9*32+15) /* Resource Director Technology Allocation */ +#define X86_FEATURE_AVX512F ( 9*32+16) /* AVX-512 Foundation */ +#define X86_FEATURE_AVX512DQ ( 9*32+17) /* AVX-512 DQ (Double/Quad granular) Instructions */ +#define X86_FEATURE_RDSEED ( 9*32+18) /* RDSEED instruction */ +#define X86_FEATURE_ADX ( 9*32+19) /* ADCX and ADOX instructions */ +#define X86_FEATURE_SMAP ( 9*32+20) /* Supervisor Mode Access Prevention */ +#define X86_FEATURE_AVX512IFMA ( 9*32+21) /* AVX-512 Integer Fused Multiply-Add instructions */ +#define X86_FEATURE_CLFLUSHOPT ( 9*32+23) /* CLFLUSHOPT instruction */ +#define X86_FEATURE_CLWB ( 9*32+24) /* CLWB instruction */ +#define X86_FEATURE_INTEL_PT ( 9*32+25) /* Intel Processor Trace */ +#define X86_FEATURE_AVX512PF ( 9*32+26) /* AVX-512 Prefetch */ +#define X86_FEATURE_AVX512ER ( 9*32+27) /* AVX-512 Exponential and Reciprocal */ +#define X86_FEATURE_AVX512CD ( 9*32+28) /* AVX-512 Conflict Detection */ +#define X86_FEATURE_SHA_NI ( 9*32+29) /* SHA1/SHA256 Instruction Extensions */ +#define X86_FEATURE_AVX512BW ( 9*32+30) /* AVX-512 BW (Byte/Word granular) Instructions */ +#define X86_FEATURE_AVX512VL ( 9*32+31) /* AVX-512 VL (128/256 Vector Length) Extensions */ -/* Intel-defined CPU QoS Sub-leaf, CPUID level 0x0000000F:0 (edx), word 11 */ -#define X86_FEATURE_CQM_LLC (11*32+ 1) /* LLC QoS if 1 */ +/* Extended state features, CPUID level 0x0000000d:1 (EAX), word 10 */ +#define X86_FEATURE_XSAVEOPT (10*32+ 0) /* XSAVEOPT instruction */ +#define X86_FEATURE_XSAVEC (10*32+ 1) /* XSAVEC instruction */ +#define X86_FEATURE_XGETBV1 (10*32+ 2) /* XGETBV with ECX = 1 instruction */ +#define X86_FEATURE_XSAVES (10*32+ 3) /* XSAVES/XRSTORS instructions */ -/* Intel-defined CPU QoS Sub-leaf, CPUID level 0x0000000F:1 (edx), word 12 */ -#define X86_FEATURE_CQM_OCCUP_LLC (12*32+ 0) /* LLC occupancy monitoring if 1 */ -#define X86_FEATURE_CQM_MBM_TOTAL (12*32+ 1) /* LLC Total MBM monitoring */ -#define X86_FEATURE_CQM_MBM_LOCAL (12*32+ 2) /* LLC Local MBM monitoring */ +/* + * Extended auxiliary flags: Linux defined - for features scattered in various + * CPUID levels like 0xf, etc. + * + * Reuse free bits when adding new feature flags! + */ +#define X86_FEATURE_CQM_LLC (11*32+ 0) /* LLC QoS if 1 */ +#define X86_FEATURE_CQM_OCCUP_LLC (11*32+ 1) /* LLC occupancy monitoring */ +#define X86_FEATURE_CQM_MBM_TOTAL (11*32+ 2) /* LLC Total MBM monitoring */ +#define X86_FEATURE_CQM_MBM_LOCAL (11*32+ 3) /* LLC Local MBM monitoring */ +#define X86_FEATURE_FENCE_SWAPGS_USER (11*32+ 4) /* "" LFENCE in user entry SWAPGS path */ +#define X86_FEATURE_FENCE_SWAPGS_KERNEL (11*32+ 5) /* "" LFENCE in kernel entry SWAPGS path */ +#define X86_FEATURE_ENTRY_IBPB (11*32+10) /* "" Issue an IBPB on kernel entry */ +#define X86_FEATURE_RRSBA_CTRL (11*32+11) /* "" RET prediction control */ +/* FREE! (11*32+12) */ +#define X86_FEATURE_RETPOLINE_AMD (11*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */ +#define X86_FEATURE_RETHUNK (11*32+14) /* "" Use REturn THUNK */ +#define X86_FEATURE_UNRET (11*32+15) /* "" AMD BTB untrain return */ /* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13 */ -#define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */ -#define X86_FEATURE_IRPERF (13*32+ 1) /* Instructions Retired Count */ -#define X86_FEATURE_XSAVEERPTR (13*32+ 2) /* Always save/restore FP error pointers */ +#define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */ +#define X86_FEATURE_IRPERF (13*32+ 1) /* Instructions Retired Count */ +#define X86_FEATURE_XSAVEERPTR (13*32+ 2) /* Always save/restore FP error pointers */ #define X86_FEATURE_AMD_IBPB (13*32+12) /* "" Indirect Branch Prediction Barrier */ #define X86_FEATURE_AMD_IBRS (13*32+14) /* "" Indirect Branch Restricted Speculation */ #define X86_FEATURE_AMD_STIBP (13*32+15) /* "" Single Thread Indirect Branch Predictors */ #define X86_FEATURE_AMD_SSBD (13*32+24) /* "" Speculative Store Bypass Disable */ #define X86_FEATURE_VIRT_SSBD (13*32+25) /* Virtualized Speculative Store Bypass Disable */ #define X86_FEATURE_AMD_SSB_NO (13*32+26) /* "" Speculative Store Bypass is fixed in hardware. */ +#define X86_FEATURE_BTC_NO (13*32+29) /* "" Not vulnerable to Branch Type Confusion */ -/* AMD-defined CPU features, CPUID level 0x80000007 (ebx), word 17 */ -#define X86_FEATURE_OVERFLOW_RECOV (17*32+0) /* MCA overflow recovery support */ -#define X86_FEATURE_SUCCOR (17*32+1) /* Uncorrectable error containment and recovery */ -#define X86_FEATURE_SMCA (17*32+3) /* Scalable MCA */ - -/* Thermal and Power Management Leaf, CPUID level 0x00000006 (eax), word 14 */ -#define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */ -#define X86_FEATURE_IDA (14*32+ 1) /* Intel Dynamic Acceleration */ -#define X86_FEATURE_ARAT (14*32+ 2) /* Always Running APIC Timer */ -#define X86_FEATURE_PLN (14*32+ 4) /* Intel Power Limit Notification */ -#define X86_FEATURE_PTS (14*32+ 6) /* Intel Package Thermal Status */ -#define X86_FEATURE_HWP (14*32+ 7) /* Intel Hardware P-states */ -#define X86_FEATURE_HWP_NOTIFY (14*32+ 8) /* HWP Notification */ -#define X86_FEATURE_HWP_ACT_WINDOW (14*32+ 9) /* HWP Activity Window */ -#define X86_FEATURE_HWP_EPP (14*32+10) /* HWP Energy Perf. Preference */ -#define X86_FEATURE_HWP_PKG_REQ (14*32+11) /* HWP Package Level Request */ +/* Thermal and Power Management Leaf, CPUID level 0x00000006 (EAX), word 14 */ +#define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */ +#define X86_FEATURE_IDA (14*32+ 1) /* Intel Dynamic Acceleration */ +#define X86_FEATURE_ARAT (14*32+ 2) /* Always Running APIC Timer */ +#define X86_FEATURE_PLN (14*32+ 4) /* Intel Power Limit Notification */ +#define X86_FEATURE_PTS (14*32+ 6) /* Intel Package Thermal Status */ +#define X86_FEATURE_HWP (14*32+ 7) /* Intel Hardware P-states */ +#define X86_FEATURE_HWP_NOTIFY (14*32+ 8) /* HWP Notification */ +#define X86_FEATURE_HWP_ACT_WINDOW (14*32+ 9) /* HWP Activity Window */ +#define X86_FEATURE_HWP_EPP (14*32+10) /* HWP Energy Perf. Preference */ +#define X86_FEATURE_HWP_PKG_REQ (14*32+11) /* HWP Package Level Request */ -/* AMD SVM Feature Identification, CPUID level 0x8000000a (edx), word 15 */ -#define X86_FEATURE_NPT (15*32+ 0) /* Nested Page Table support */ -#define X86_FEATURE_LBRV (15*32+ 1) /* LBR Virtualization support */ -#define X86_FEATURE_SVML (15*32+ 2) /* "svm_lock" SVM locking MSR */ -#define X86_FEATURE_NRIPS (15*32+ 3) /* "nrip_save" SVM next_rip save */ -#define X86_FEATURE_TSCRATEMSR (15*32+ 4) /* "tsc_scale" TSC scaling support */ -#define X86_FEATURE_VMCBCLEAN (15*32+ 5) /* "vmcb_clean" VMCB clean bits support */ -#define X86_FEATURE_FLUSHBYASID (15*32+ 6) /* flush-by-ASID support */ -#define X86_FEATURE_DECODEASSISTS (15*32+ 7) /* Decode Assists support */ -#define X86_FEATURE_PAUSEFILTER (15*32+10) /* filtered pause intercept */ -#define X86_FEATURE_PFTHRESHOLD (15*32+12) /* pause filter threshold */ -#define X86_FEATURE_AVIC (15*32+13) /* Virtual Interrupt Controller */ -#define X86_FEATURE_V_VMSAVE_VMLOAD (15*32+15) /* Virtual VMSAVE VMLOAD */ -#define X86_FEATURE_VGIF (15*32+16) /* Virtual GIF */ +/* AMD SVM Feature Identification, CPUID level 0x8000000a (EDX), word 15 */ +#define X86_FEATURE_NPT (15*32+ 0) /* Nested Page Table support */ +#define X86_FEATURE_LBRV (15*32+ 1) /* LBR Virtualization support */ +#define X86_FEATURE_SVML (15*32+ 2) /* "svm_lock" SVM locking MSR */ +#define X86_FEATURE_NRIPS (15*32+ 3) /* "nrip_save" SVM next_rip save */ +#define X86_FEATURE_TSCRATEMSR (15*32+ 4) /* "tsc_scale" TSC scaling support */ +#define X86_FEATURE_VMCBCLEAN (15*32+ 5) /* "vmcb_clean" VMCB clean bits support */ +#define X86_FEATURE_FLUSHBYASID (15*32+ 6) /* flush-by-ASID support */ +#define X86_FEATURE_DECODEASSISTS (15*32+ 7) /* Decode Assists support */ +#define X86_FEATURE_PAUSEFILTER (15*32+10) /* filtered pause intercept */ +#define X86_FEATURE_PFTHRESHOLD (15*32+12) /* pause filter threshold */ +#define X86_FEATURE_AVIC (15*32+13) /* Virtual Interrupt Controller */ +#define X86_FEATURE_V_VMSAVE_VMLOAD (15*32+15) /* Virtual VMSAVE VMLOAD */ +#define X86_FEATURE_VGIF (15*32+16) /* Virtual GIF */ -/* Intel-defined CPU features, CPUID level 0x00000007:0 (ecx), word 16 */ -#define X86_FEATURE_AVX512VBMI (16*32+ 1) /* AVX512 Vector Bit Manipulation instructions*/ -#define X86_FEATURE_UMIP (16*32+ 2) /* User Mode Instruction Protection */ -#define X86_FEATURE_PKU (16*32+ 3) /* Protection Keys for Userspace */ -#define X86_FEATURE_OSPKE (16*32+ 4) /* OS Protection Keys Enable */ -#define X86_FEATURE_AVX512_VBMI2 (16*32+ 6) /* Additional AVX512 Vector Bit Manipulation Instructions */ -#define X86_FEATURE_GFNI (16*32+ 8) /* Galois Field New Instructions */ -#define X86_FEATURE_VAES (16*32+ 9) /* Vector AES */ -#define X86_FEATURE_VPCLMULQDQ (16*32+ 10) /* Carry-Less Multiplication Double Quadword */ -#define X86_FEATURE_AVX512_VNNI (16*32+ 11) /* Vector Neural Network Instructions */ -#define X86_FEATURE_AVX512_BITALG (16*32+12) /* Support for VPOPCNT[B,W] and VPSHUF-BITQMB */ -#define X86_FEATURE_AVX512_VPOPCNTDQ (16*32+14) /* POPCNT for vectors of DW/QW */ +/* Intel-defined CPU features, CPUID level 0x00000007:0 (ECX), word 16 */ +#define X86_FEATURE_AVX512VBMI (16*32+ 1) /* AVX512 Vector Bit Manipulation instructions*/ +#define X86_FEATURE_UMIP (16*32+ 2) /* User Mode Instruction Protection */ +#define X86_FEATURE_PKU (16*32+ 3) /* Protection Keys for Userspace */ +#define X86_FEATURE_OSPKE (16*32+ 4) /* OS Protection Keys Enable */ +#define X86_FEATURE_AVX512_VBMI2 (16*32+ 6) /* Additional AVX512 Vector Bit Manipulation Instructions */ +#define X86_FEATURE_GFNI (16*32+ 8) /* Galois Field New Instructions */ +#define X86_FEATURE_VAES (16*32+ 9) /* Vector AES */ +#define X86_FEATURE_VPCLMULQDQ (16*32+10) /* Carry-Less Multiplication Double Quadword */ +#define X86_FEATURE_AVX512_VNNI (16*32+11) /* Vector Neural Network Instructions */ +#define X86_FEATURE_AVX512_BITALG (16*32+12) /* Support for VPOPCNT[B,W] and VPSHUF-BITQMB instructions */ +#define X86_FEATURE_AVX512_VPOPCNTDQ (16*32+14) /* POPCNT for vectors of DW/QW */ #define X86_FEATURE_CLDEMOTE (16*32+25) /* CLDEMOTE instruction */ #define X86_FEATURE_MOVDIRI (16*32+27) /* MOVDIRI instruction */ #define X86_FEATURE_MOVDIR64B (16*32+28) /* MOVDIR64B instruction */ +/* AMD-defined CPU features, CPUID level 0x80000007 (EBX), word 17 */ +#define X86_FEATURE_OVERFLOW_RECOV (17*32+ 0) /* MCA overflow recovery support */ +#define X86_FEATURE_SUCCOR (17*32+ 1) /* Uncorrectable error containment and recovery */ +#define X86_FEATURE_SMCA (17*32+ 3) /* Scalable MCA */ + /* Intel-defined CPU features, CPUID level 0x00000007:0 (EDX), word 18 */ #define X86_FEATURE_AVX512_4VNNIW (18*32+ 2) /* AVX-512 Neural Network Instructions */ #define X86_FEATURE_AVX512_4FMAPS (18*32+ 3) /* AVX-512 Multiply Accumulation Single precision */ @@ -367,5 +378,6 @@ #define X86_BUG_ITLB_MULTIHIT X86_BUG(23) /* CPU may incur MCE during certain page attribute changes */ #define X86_BUG_SRBDS X86_BUG(24) /* CPU may leak RNG bits if not mitigated */ #define X86_BUG_MMIO_STALE_DATA X86_BUG(25) /* CPU is affected by Processor MMIO Stale Data vulnerabilities */ +#define X86_BUG_RETBLEED X86_BUG(26) /* CPU is affected by RETBleed */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h index 476b053ef1c1e3..87fdfaab83cb3c 100644 --- a/arch/x86/include/asm/disabled-features.h +++ b/arch/x86/include/asm/disabled-features.h @@ -42,6 +42,14 @@ # define DISABLE_OSPKE (1<<(X86_FEATURE_OSPKE & 31)) #endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */ +#ifndef CONFIG_RETPOLINE +# define DISABLE_RETPOLINE 0 +#else +# define DISABLE_RETPOLINE ((1 << (X86_FEATURE_RETPOLINE_AMD & 31)) | \ + (1 << (X86_FEATURE_RETHUNK & 31)) | \ + (1 << (X86_FEATURE_UNRET & 31))) +#endif + /* * Make sure to add features to the correct mask */ @@ -56,7 +64,7 @@ #define DISABLED_MASK8 0 #define DISABLED_MASK9 (DISABLE_MPX) #define DISABLED_MASK10 0 -#define DISABLED_MASK11 0 +#define DISABLED_MASK11 (DISABLE_RETPOLINE) #define DISABLED_MASK12 0 #define DISABLED_MASK13 0 #define DISABLED_MASK14 0 diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index 79327e9483a34e..55d5b19bdf3430 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -55,6 +55,20 @@ #define __ALIGN_STR __stringify(__ALIGN) #endif +#if defined(CONFIG_RETPOLINE) && !defined(BUILD_VDSO) +#define RET jmp __x86_return_thunk +#else +#define RET ret +#endif + +#else /* __ASSEMBLY__ */ + +#if defined(CONFIG_RETPOLINE) && !defined(BUILD_VDSO) +#define ASM_RET "jmp __x86_return_thunk\n\t" +#else +#define ASM_RET "ret\n\t" +#endif + #endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_LINKAGE_H */ diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 0607fd9388b349..38110791185131 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -2,8 +2,12 @@ #define _ASM_X86_MSR_INDEX_H #ifndef BIT +#ifdef __ASSEMBLY__ +#define BIT(nr) (1 << (nr)) +#else #define BIT(nr) (1UL << (nr)) #endif +#endif /* CPU model specific register (MSR) numbers */ @@ -43,6 +47,8 @@ #define SPEC_CTRL_STIBP BIT(SPEC_CTRL_STIBP_SHIFT) /* STIBP mask */ #define SPEC_CTRL_SSBD_SHIFT 2 /* Speculative Store Bypass Disable bit */ #define SPEC_CTRL_SSBD BIT(SPEC_CTRL_SSBD_SHIFT) /* Speculative Store Bypass Disable */ +#define SPEC_CTRL_RRSBA_DIS_S_SHIFT 6 /* Disable RRSBA behavior */ +#define SPEC_CTRL_RRSBA_DIS_S BIT(SPEC_CTRL_RRSBA_DIS_S_SHIFT) #define MSR_IA32_PRED_CMD 0x00000049 /* Prediction Command */ #define PRED_CMD_IBPB BIT(0) /* Indirect Branch Prediction Barrier */ @@ -69,6 +75,7 @@ #define MSR_IA32_ARCH_CAPABILITIES 0x0000010a #define ARCH_CAP_RDCL_NO BIT(0) /* Not susceptible to Meltdown */ #define ARCH_CAP_IBRS_ALL BIT(1) /* Enhanced IBRS support */ +#define ARCH_CAP_RSBA BIT(2) /* RET may use alternative branch predictors */ #define ARCH_CAP_SKIP_VMENTRY_L1DFLUSH BIT(3) /* Skip L1D flush on vmentry */ #define ARCH_CAP_SSB_NO BIT(4) /* * Not susceptible to Speculative Store Bypass @@ -116,6 +123,13 @@ * bit available to control VERW * behavior. */ +#define ARCH_CAP_RRSBA BIT(19) /* + * Indicates RET may use predictors + * other than the RSB. With eIBRS + * enabled predictions in kernel mode + * are restricted to targets in + * kernel. + */ #define MSR_IA32_FLUSH_CMD 0x0000010b #define L1D_FLUSH BIT(0) /* @@ -404,6 +418,9 @@ #define MSR_AMD64_SEV_ENABLED_BIT 0 #define MSR_AMD64_SEV_ENABLED BIT_ULL(MSR_AMD64_SEV_ENABLED_BIT) +#define MSR_ZEN2_SPECTRAL_CHICKEN 0xc00110e3 +#define MSR_ZEN2_SPECTRAL_CHICKEN_BIT BIT_ULL(1) + /* Fam 16h MSRs */ #define MSR_F16H_L2I_PERF_CTL 0xc0010230 #define MSR_F16H_L2I_PERF_CTR 0xc0010231 diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 7f927c4692a2d0..7f9549fa998ea3 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -51,9 +51,11 @@ lfence; \ jmp 775b; \ 774: \ + add $(BITS_PER_LONG/8) * 2, sp; \ dec reg; \ jnz 771b; \ - add $(BITS_PER_LONG/8) * nr, sp; + /* barrier for jnz misprediction */ \ + lfence; #ifdef __ASSEMBLY__ @@ -61,20 +63,21 @@ * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP * monstrosity above, manually. */ -.macro FILL_RETURN_BUFFER_CLOBBER reg=%rax - 661: __FILL_RETURN_BUFFER(\reg, RSB_CLEAR_LOOPS, %_ASM_SP); 662: +.macro FILL_RETURN_BUFFER_CLOBBER reg:req ftr:req + 661: jmp .Lskip_rsb_\@; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP4; 662: .pushsection .altinstr_replacement, "ax" - 663: ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP3; 664: + 663: __FILL_RETURN_BUFFER(\reg, RSB_CLEAR_LOOPS, %_ASM_SP); 664: .popsection .pushsection .altinstructions, "a" - altinstruction_entry 661b, 663b, X86_FEATURE_SMEP, 662b-661b, 664b-663b + altinstruction_entry 661b, 663b, \ftr, 662b-661b, 664b-663b .popsection +.Lskip_rsb_\@: .endm -.macro FILL_RETURN_BUFFER - push %rax - FILL_RETURN_BUFFER_CLOBBER reg=%rax - pop %rax +.macro FILL_RETURN_BUFFER reg:req ftr:req + push \reg + FILL_RETURN_BUFFER_CLOBBER reg=\reg ftr=\ftr + pop \reg .endm /* @@ -166,8 +169,37 @@ .Ldone_\@: .endm +/* + * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the + * return thunk isn't mapped into the userspace tables (then again, AMD + * typically has NO_MELTDOWN). + * + * While zen_untrain_ret() doesn't clobber anything but requires stack, + * entry_ibpb() will clobber AX, CX, DX. + * + * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point + * where we have a stack but before any RET instruction. + */ +.macro UNTRAIN_RET +#ifdef CONFIG_RETPOLINE + 661: ASM_NOP5_ATOMIC; 662: + .pushsection .altinstr_replacement, "ax" + 6631: call zen_untrain_ret; 6641: + 6632: call entry_ibpb; 6642: + .popsection + .pushsection .altinstructions, "a" + altinstruction_entry 661b, 6631b, X86_FEATURE_UNRET, 662b-661b, 6641b-6631b + altinstruction_entry 661b, 6632b, X86_FEATURE_ENTRY_IBPB, 662b-661b, 6642b-6632b + .popsection +#endif +.endm + #else /* __ASSEMBLY__ */ +extern void __x86_return_thunk(void); +extern void zen_untrain_ret(void); +extern void entry_ibpb(void); + #if defined(CONFIG_X86_64) && defined(RETPOLINE) /* * Since the inline asm uses the %V modifier which is only in newer GCC, @@ -248,6 +280,20 @@ static inline void fill_RSB(void) : : "memory" ); } +/* + * On VMEXIT we must ensure we untrain RET in order to mitigate + * RETBleed for AMD/Hygon Zen uarch. + */ +static inline void untrain_ret(void) +{ + asm volatile ( + ALTERNATIVE_2(ASM_NOP5_ATOMIC, + "call zen_untrain_ret", X86_FEATURE_UNRET, + "call entry_ibpb", X86_FEATURE_ENTRY_IBPB) + : : : "memory" + ); +} + extern struct static_key mds_user_clear; extern struct static_key mds_idle_clear; extern struct static_key mmio_stale_data_clear; diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index b62cb929717350..994f2b1616787a 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -769,7 +769,7 @@ static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock, "call " #func ";" \ PV_RESTORE_ALL_CALLER_REGS \ FRAME_END \ - "ret;" \ + ASM_RET \ ".popsection") /* Get a reference to a callee-save function */ diff --git a/arch/x86/include/asm/qspinlock_paravirt.h b/arch/x86/include/asm/qspinlock_paravirt.h index 9d55f9b6e167c1..149ff1e5888aa6 100644 --- a/arch/x86/include/asm/qspinlock_paravirt.h +++ b/arch/x86/include/asm/qspinlock_paravirt.h @@ -48,7 +48,7 @@ asm (".pushsection .text;" "jne .slowpath;" "pop %rdx;" FRAME_END - "ret;" + ASM_RET ".slowpath: " "push %rsi;" "movzbl %al,%esi;" @@ -56,7 +56,7 @@ asm (".pushsection .text;" "pop %rsi;" "pop %rdx;" FRAME_END - "ret;" + ASM_RET ".size " PV_UNLOCK ", .-" PV_UNLOCK ";" ".popsection"); diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 7a3b4c29a083ce..b134167cad7a90 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -24,7 +24,7 @@ OBJECT_FILES_NON_STANDARD_entry_$(BITS).o := y CFLAGS_irq.o := -I$(src)/../include/asm/trace -obj-y := process_$(BITS).o signal.o entry_$(BITS).o +obj-y := process_$(BITS).o signal.o entry_$(BITS).o entry.o obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o ldt.o dumpstack.o nmi.o obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o diff --git a/arch/x86/kernel/acpi/wakeup_32.S b/arch/x86/kernel/acpi/wakeup_32.S index 665c6b7d2ea93c..a7fda87fcc1660 100644 --- a/arch/x86/kernel/acpi/wakeup_32.S +++ b/arch/x86/kernel/acpi/wakeup_32.S @@ -57,7 +57,7 @@ save_registers: popl saved_context_eflags movl $ret_point, saved_eip - ret + RET restore_registers: @@ -67,7 +67,7 @@ restore_registers: movl saved_context_edi, %edi pushl saved_context_eflags popfl - ret + RET ENTRY(do_suspend_lowlevel) call save_processor_state @@ -83,7 +83,7 @@ ENTRY(do_suspend_lowlevel) ret_point: call restore_registers call restore_processor_state - ret + RET .data ALIGN diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 137dcbca27ba08..a7aec4402e5e6b 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -42,12 +42,28 @@ static int __init setup_noreplace_smp(char *str) } __setup("noreplace-smp", setup_noreplace_smp); -#define DPRINTK(fmt, ...) \ -do { \ - if (debug_alternative) \ - printk(KERN_DEBUG fmt, ##__VA_ARGS__); \ +#define DPRINTK(fmt, args...) \ +do { \ + if (debug_alternative) \ + printk(KERN_DEBUG "%s: " fmt "\n", __func__, ##args); \ } while (0) +#define DUMP_BYTES(buf, len, fmt, args...) \ +do { \ + if (unlikely(debug_alternative)) { \ + int j; \ + \ + if (!(len)) \ + break; \ + \ + printk(KERN_DEBUG pr_fmt(fmt), ##args); \ + for (j = 0; j < (len) - 1; j++) \ + printk(KERN_CONT "%02hhx ", buf[j]); \ + printk(KERN_CONT "%02hhx\n", buf[j]); \ + } \ +} while (0) + + /* * Each GENERIC_NOPX is of X bytes, and defined as an array of bytes * that correspond to that nop. Getting from one nop to the next, we @@ -229,16 +245,18 @@ static void __init_or_module add_nops(void *insns, unsigned int len) } } +extern s32 __return_sites[], __return_sites_end[]; extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; extern s32 __smp_locks[], __smp_locks_end[]; void *text_poke_early(void *addr, const void *opcode, size_t len); -/* Replace instructions with better alternatives for this CPU type. - This runs before SMP is initialized to avoid SMP problems with - self modifying code. This implies that asymmetric systems where - APs have less capabilities than the boot processor are not handled. - Tough. Make sure you disable such features by hand. */ - +/* + * Replace instructions with better alternatives for this CPU type. This runs + * before SMP is initialized to avoid SMP problems with self modifying code. + * This implies that asymmetric systems where APs have less capabilities than + * the boot processor are not handled. Tough. Make sure you disable such + * features by hand. + */ void __init_or_module apply_alternatives(struct alt_instr *start, struct alt_instr *end) { @@ -246,10 +264,10 @@ void __init_or_module apply_alternatives(struct alt_instr *start, u8 *instr, *replacement; u8 insnbuf[MAX_PATCH_LEN]; - DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); + DPRINTK("alt table %p -> %p", start, end); /* * The scan order should be from start to end. A later scanned - * alternative code can overwrite a previous scanned alternative code. + * alternative code can overwrite previously scanned alternative code. * Some kernel functions (e.g. memcpy, memset, etc) use this order to * patch code. * @@ -265,11 +283,19 @@ void __init_or_module apply_alternatives(struct alt_instr *start, if (!boot_cpu_has(a->cpuid)) continue; + DPRINTK("feat: %d*32+%d, old: (%p, len: %d), repl: (%p, len: %d)", + a->cpuid >> 5, + a->cpuid & 0x1f, + instr, a->instrlen, + replacement, a->replacementlen); + memcpy(insnbuf, replacement, a->replacementlen); /* 0xe8 is a relative jump; fix the offset. */ - if (*insnbuf == 0xe8 && a->replacementlen == 5) - *(s32 *)(insnbuf + 1) += replacement - instr; + if (*insnbuf == 0xe8 && a->replacementlen == 5) { + *(s32 *)(insnbuf + 1) += replacement - instr; + DPRINTK("Fix CALL offset: 0x%x", *(s32 *)(insnbuf + 1)); + } add_nops(insnbuf + a->replacementlen, a->instrlen - a->replacementlen); @@ -361,8 +387,8 @@ void __init_or_module alternatives_smp_module_add(struct module *mod, smp->locks_end = locks_end; smp->text = text; smp->text_end = text_end; - DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n", - __func__, smp->locks, smp->locks_end, + DPRINTK("locks %p -> %p, text %p -> %p, name %s\n", + smp->locks, smp->locks_end, smp->text, smp->text_end, smp->name); list_add_tail(&smp->next, &smp_alt_modules); @@ -470,6 +496,73 @@ extern struct paravirt_patch_site __start_parainstructions[], __stop_parainstructions[]; #endif /* CONFIG_PARAVIRT */ +#define INT3_INSN_OPCODE 0xCC +#define RET_INSN_OPCODE 0xC3 +#define JMP32_INSN_OPCODE 0xE9 + +/* + * Rewrite the compiler generated return thunk tail-calls. + * + * For example, convert: + * + * JMP __x86_return_thunk + * + * into: + * + * RET + */ +static int patch_return(void *addr, struct insn *insn, u8 *bytes) +{ + int i = 0; + + if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) + return -1; + + bytes[i++] = RET_INSN_OPCODE; + + for (; i < insn->length;) + bytes[i++] = INT3_INSN_OPCODE; + + return i; +} + +void __init_or_module noinline apply_returns(s32 *start, s32 *end) +{ + s32 *s; + + DPRINTK("return thunk table %p -> %p", start, end); + + for (s = start; s < end; s++) { + void *addr = (void *)s + *s; + struct insn insn; + int len, ret; + u8 bytes[16]; + u8 op1; + + insn_init(&insn, addr, MAX_INSN_SIZE, IS_ENABLED(CONFIG_X86_64)); + insn_get_length(&insn); + + ret = insn_complete(&insn); + if (WARN_ON_ONCE(!ret)) + continue; + + op1 = insn.opcode.bytes[0]; + if (WARN_ON_ONCE(op1 != JMP32_INSN_OPCODE)) + continue; + + DPRINTK("return thunk at: %pS (%px) len: %d to: %pS", + addr, addr, insn.length, + addr + insn.length + insn.immediate.value); + + len = patch_return(addr, &insn, bytes); + if (len == insn.length) { + DUMP_BYTES(((u8*)addr), len, "%px: orig: ", addr); + DUMP_BYTES(((u8*)bytes), len, "%px: repl: ", addr); + text_poke_early(addr, bytes, len); + } + } +} + void __init alternative_instructions(void) { /* The patching is not fully atomic, so try to avoid local interruptions @@ -477,6 +570,13 @@ void __init alternative_instructions(void) Other CPUs are not running. */ stop_nmi(); + + /* + * Rewrite return thunks before applying the alternatives as + * those can rewrite the thunks + */ + apply_returns(__return_sites, __return_sites_end); + /* * Don't stop machine check exceptions while patching. * MCEs only happen when something got corrupted and in this diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index d7ce06da0b0b89..93d41928ebd549 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -641,6 +641,26 @@ static const int amd_erratum_383[]; static const int amd_erratum_400[]; static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum); +void init_spectral_chicken(struct cpuinfo_x86 *c) +{ + u64 value; + + /* + * On Zen2 we offer this chicken (bit) on the altar of Speculation. + * + * This suppresses speculation from the middle of a basic block, i.e. it + * suppresses non-branch predictions. + * + * We use STIBP as a heuristic to filter out Zen2 from the rest of F17H + */ + if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && cpu_has(c, X86_FEATURE_AMD_STIBP)) { + if (!rdmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, &value)) { + value |= MSR_ZEN2_SPECTRAL_CHICKEN_BIT; + wrmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, value); + } + } +} + static void init_amd_zn(struct cpuinfo_x86 *c) { set_cpu_cap(c, X86_FEATURE_ZEN); @@ -649,12 +669,24 @@ static void init_amd_zn(struct cpuinfo_x86 *c) node_reclaim_distance = 32; #endif - /* - * Fix erratum 1076: CPB feature bit not being set in CPUID. It affects - * all up to and including B1. - */ - if (c->x86_model <= 1 && c->x86_mask <= 1) - set_cpu_cap(c, X86_FEATURE_CPB); + /* Fix up CPUID bits, but only if not virtualised. */ + if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) { + + /* + * Fix erratum 1076: CPB feature bit not being set in CPUID. It affects + * all up to and including B1. + */ + if (c->x86_model <= 1 && c->x86_mask <= 1) + set_cpu_cap(c, X86_FEATURE_CPB); + + /* + * Zen3 (Fam19 model < 0x10) parts are not susceptible to + * Branch Type Confusion, but predate the allocation of the + * BTC_NO bit. + */ + if (c->x86 == 0x19 && !cpu_has(c, X86_FEATURE_BTC_NO)) + set_cpu_cap(c, X86_FEATURE_BTC_NO); + } } static void init_amd(struct cpuinfo_x86 *c) @@ -713,6 +745,8 @@ static void init_amd(struct cpuinfo_x86 *c) c->apicid = hard_smp_processor_id(); switch (c->x86) { + case 0x17: init_spectral_chicken(c); + /* fallthrough */ case 0x19: init_amd_zn(c); break; } #else diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index a7e93ef8174564..5546d93b0160b1 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -32,6 +32,7 @@ #include "cpu.h" static void __init spectre_v1_select_mitigation(void); +static void __init retbleed_select_mitigation(void); static void __init spectre_v2_select_mitigation(void); static void __init ssb_parse_cmdline(void); void ssb_select_mitigation(void); @@ -87,6 +88,7 @@ void __init check_bugs(void) /* Select the proper CPU mitigations before patching alternatives */ spectre_v1_select_mitigation(); spectre_v2_select_mitigation(); + retbleed_select_mitigation(); spec_ctrl_cpu_init(); ssb_select_mitigation(); l1tf_select_mitigation(); @@ -147,6 +149,7 @@ enum spectre_v2_mitigation_cmd { SPECTRE_V2_CMD_FORCE, SPECTRE_V2_CMD_AUTO, SPECTRE_V2_CMD_RETPOLINE, + SPECTRE_V2_CMD_RETPOLINE_FORCE, SPECTRE_V2_CMD_RETPOLINE_IBRS_USER, SPECTRE_V2_CMD_IBRS, SPECTRE_V2_CMD_IBRS_ALWAYS, @@ -649,10 +652,166 @@ static int __init nospectre_v1_cmdline(char *str) } early_param("nospectre_v1", nospectre_v1_cmdline); +enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE; + #undef pr_fmt -#define pr_fmt(fmt) "Spectre V2 : " fmt +#define pr_fmt(fmt) "RETBleed: " fmt + +enum retbleed_mitigation { + RETBLEED_MITIGATION_NONE, + RETBLEED_MITIGATION_UNRET, + RETBLEED_MITIGATION_IBPB, + RETBLEED_MITIGATION_IBRS, + RETBLEED_MITIGATION_EIBRS, +}; -enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE; +enum retbleed_mitigation_cmd { + RETBLEED_CMD_OFF, + RETBLEED_CMD_AUTO, + RETBLEED_CMD_UNRET, + RETBLEED_CMD_IBPB, +}; + +const char * const retbleed_strings[] = { + [RETBLEED_MITIGATION_NONE] = "Vulnerable", + [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", + [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB", + [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS", + [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS", +}; + +static enum retbleed_mitigation retbleed_mitigation __read_mostly = + RETBLEED_MITIGATION_NONE; +static enum retbleed_mitigation_cmd retbleed_cmd __read_mostly = + RETBLEED_CMD_AUTO; + +static int __read_mostly retbleed_nosmt = false; + +static int __init retbleed_parse_cmdline(char *str) +{ + while (str) { + char *next = strchr(str, ','); + if (next) { + *next = 0; + next++; + } + + if (!strcmp(str, "off")) { + retbleed_cmd = RETBLEED_CMD_OFF; + } else if (!strcmp(str, "auto")) { + retbleed_cmd = RETBLEED_CMD_AUTO; + } else if (!strcmp(str, "unret")) { + retbleed_cmd = RETBLEED_CMD_UNRET; + } else if (!strcmp(str, "ibpb")) { + retbleed_cmd = RETBLEED_CMD_IBPB; + } else if (!strcmp(str, "nosmt")) { + retbleed_nosmt = true; + } else { + pr_err("Ignoring unknown retbleed option (%s).", str); + } + + str = next; + } + + return 0; +} +early_param("retbleed", retbleed_parse_cmdline); + +#define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD!\n" +#define RETBLEED_COMPILER_MSG "WARNING: kernel not compiled with RETPOLINE or -mfunction-return capable compiler; falling back to IBPB!\n" +#define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n" + +static void __init retbleed_select_mitigation(void) +{ + bool mitigate_smt = false; + + if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) + return; + + switch (retbleed_cmd) { + case RETBLEED_CMD_OFF: + return; + + case RETBLEED_CMD_UNRET: + retbleed_mitigation = RETBLEED_MITIGATION_UNRET; + break; + + case RETBLEED_CMD_IBPB: + if (boot_cpu_has(X86_FEATURE_IBPB)) { + retbleed_mitigation = RETBLEED_MITIGATION_IBPB; + break; + } else { + pr_err("WARNING: CPU does not support IBPB.\n"); + } + /* fallthrough */ + case RETBLEED_CMD_AUTO: + default: + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { + if (!IS_ENABLED(CONFIG_RETPOLINE) && boot_cpu_has(X86_FEATURE_IBPB)) { + pr_err(RETBLEED_COMPILER_MSG); + retbleed_mitigation = RETBLEED_MITIGATION_IBPB; + } else { + retbleed_mitigation = RETBLEED_MITIGATION_UNRET; + } + } + + /* + * The Intel mitigation (IBRS or eIBRS) was already selected in + * spectre_v2_select_mitigation(). 'retbleed_mitigation' will + * be set accordingly below. + */ + + break; + } + + switch (retbleed_mitigation) { + case RETBLEED_MITIGATION_UNRET: + setup_force_cpu_cap(X86_FEATURE_RETHUNK); + setup_force_cpu_cap(X86_FEATURE_UNRET); + + if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) + pr_err(RETBLEED_UNTRAIN_MSG); + + mitigate_smt = true; + break; + + case RETBLEED_MITIGATION_IBPB: + setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); + mitigate_smt = true; + break; + + default: + break; + } + + if (mitigate_smt && (!boot_cpu_has(X86_FEATURE_STIBP) || + !(x86_spec_ctrl_base & SPEC_CTRL_STIBP)) && + (retbleed_nosmt || cpu_mitigations_auto_nosmt())) + cpu_smt_disable(false); + + /* + * Let IBRS trump all on Intel without affecting the effects of the + * retbleed= cmdline option. + */ + if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { + switch (spectre_v2_enabled) { + case SPECTRE_V2_IBRS_ALWAYS: + case SPECTRE_V2_IBRS: + retbleed_mitigation = RETBLEED_MITIGATION_IBRS; + break; + case SPECTRE_V2_IBRS_ENHANCED: + retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; + break; + default: + pr_err(RETBLEED_INTEL_MSG); + } + } + + pr_info("%s\n", retbleed_strings[retbleed_mitigation]); +} + +#undef pr_fmt +#define pr_fmt(fmt) "Spectre V2 : " fmt static inline bool match_option(const char *arg, int arglen, const char *opt) { @@ -669,6 +828,7 @@ static const struct { { "off", SPECTRE_V2_CMD_NONE, false }, { "on", SPECTRE_V2_CMD_FORCE, true }, { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, + { "retpoline,force", SPECTRE_V2_CMD_RETPOLINE_FORCE, false }, { "retpoline,ibrs_user",SPECTRE_V2_CMD_RETPOLINE_IBRS_USER,false }, { "ibrs", SPECTRE_V2_CMD_IBRS, false }, { "ibrs_always", SPECTRE_V2_CMD_IBRS_ALWAYS, false }, @@ -724,6 +884,30 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) return SPECTRE_V2_CMD_AUTO; } + /* + * RETBleed affected CPUs (Intel) depend on IBRS as an effective + * mitigation mechanism. We'll override spectre_v2=retpoline with + * spectre_v2=auto here, unless the old behavior is forced by + * amending ',force' to the spectre_v2 cmdline. + */ + if (boot_cpu_has_bug(X86_BUG_RETBLEED) && + boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { + switch (cmd) { + case SPECTRE_V2_CMD_RETPOLINE: + pr_warn("WARNING: %s selected but CPU is affected by RETBleed. " \ + "Switching to AUTO select\n", + mitigation_options[i].option); + return SPECTRE_V2_CMD_AUTO; + + case SPECTRE_V2_CMD_RETPOLINE_FORCE: + pr_warn("WARNING: %s selected but CPU is affected by RETBleed. " \ + "Switching to \"auto\" is advised\n", + mitigation_options[i].option); + default: + break; + } + } + if (mitigation_options[i].secure) spec2_print_if_secure(mitigation_options[i].option); else @@ -763,6 +947,7 @@ void __spectre_v2_select_mitigation(void) } break; + case SPECTRE_V2_CMD_RETPOLINE_FORCE: case SPECTRE_V2_CMD_RETPOLINE: spec_ctrl_enable_retpoline(); return; @@ -871,11 +1056,55 @@ void arch_smt_update(void) mutex_unlock(&spec_ctrl_mutex); } +/* Disable in-kernel use of non-RSB RET predictors */ +static void __init spec_ctrl_disable_kernel_rrsba(void) +{ + u64 ia32_cap; + + if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL)) + return; + + ia32_cap = x86_read_arch_cap_msr(); + + if (ia32_cap & ARCH_CAP_RRSBA) { + x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; + native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); + } +} + static void __init spectre_v2_select_mitigation(void) { + enum spectre_v2_mitigation mode; + spectre_v2_cmd = spectre_v2_parse_cmdline(); __spectre_v2_select_mitigation(); + + /* + * Disable alternate RSB predictions in kernel when indirect CALLs and + * JMPs gets protection against BHI and Intramode-BTI, but RET + * prediction from a non-RSB predictor is still a risk. + */ + barrier(); + mode = spec_ctrl_get_mitigation(); + if (mode == SPECTRE_V2_IBRS_ENHANCED || + mode == SPECTRE_V2_RETPOLINE_IBRS_USER || + mode == SPECTRE_V2_RETPOLINE) + spec_ctrl_disable_kernel_rrsba(); + spectre_v2_print_mitigation(); + + /* + * If spectre v2 protection has been enabled, unconditionally fill + * RSB during a context switch; this protects against two independent + * issues: + * + * - RSB underflow (and switch to BTB) on Skylake+ + * - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs + */ + if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && mode != SPECTRE_V2_NONE) { + setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); + pr_info("Filling RSB on context switch\n"); + } } #undef pr_fmt @@ -1393,6 +1622,22 @@ static ssize_t srbds_show_state(char *buf) return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]); } +static ssize_t retbleed_show_state(char *buf) +{ + if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET) { + if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) + return sprintf(buf, "Vulnerable: untrained return thunk on non-Zen uarch\n"); + + return sprintf(buf, "%s; SMT %s\n", + retbleed_strings[retbleed_mitigation], + !sched_smt_active() ? "disabled" : + (x86_spec_ctrl_base & SPEC_CTRL_STIBP) ? + "enabled with STIBP protection" : "vulnerable"); + } + + return sprintf(buf, "%s\n", retbleed_strings[retbleed_mitigation]); +} + static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, char *buf, unsigned int bug) { @@ -1438,6 +1683,9 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr case X86_BUG_MMIO_STALE_DATA: return mmio_stale_data_show_state(buf); + case X86_BUG_RETBLEED: + return retbleed_show_state(buf); + default: break; } @@ -1497,4 +1745,9 @@ ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *at { return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); } + +ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) +{ + return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED); +} #endif diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 4cb18c949fd427..6773edf7685ac5 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -785,6 +785,33 @@ static void apply_forced_caps(struct cpuinfo_x86 *c) } } +static void init_cqm(struct cpuinfo_x86 *c) +{ + struct rh_cpuinfo_x86 *rh_c = + (c == &boot_cpu_data) ? &rh_boot_cpu_data : &rh_cpu_data(c->cpu_index); + + if (!cpu_has(c, X86_FEATURE_CQM_LLC)) { + rh_c->x86_cache_max_rmid = -1; + rh_c->x86_cache_occ_scale = -1; + return; + } + + /* will be overridden if occupancy monitoring exists */ + rh_c->x86_cache_max_rmid = cpuid_ebx(0xf); + + if (cpu_has(c, X86_FEATURE_CQM_OCCUP_LLC) || + cpu_has(c, X86_FEATURE_CQM_MBM_TOTAL) || + cpu_has(c, X86_FEATURE_CQM_MBM_LOCAL)) { + u32 eax, ebx, ecx, edx; + + /* QoS sub-leaf, EAX=0Fh, ECX=1 */ + cpuid_count(0xf, 1, &eax, &ebx, &ecx, &edx); + + rh_c->x86_cache_max_rmid = ecx; + rh_c->x86_cache_occ_scale = ebx; + } +} + void get_cpu_cap(struct cpuinfo_x86 *c) { u32 eax, ebx, ecx, edx; @@ -797,13 +824,14 @@ void get_cpu_cap(struct cpuinfo_x86 *c) c->x86_capability[CPUID_1_EDX] = edx; } + /* Thermal and Power Management Leaf: level 0x00000006 (eax) */ + if (c->cpuid_level >= 0x00000006) + c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006); + /* Additional Intel-defined flags: level 0x00000007 */ if (c->cpuid_level >= 0x00000007) { cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx); - c->x86_capability[CPUID_7_0_EBX] = ebx; - - c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006); c->x86_capability[CPUID_7_ECX] = ecx; c->x86_capability[CPUID_7_EDX] = edx; } @@ -815,36 +843,6 @@ void get_cpu_cap(struct cpuinfo_x86 *c) c->x86_capability[CPUID_D_1_EAX] = eax; } - /* Additional Intel-defined flags: level 0x0000000F */ - if (c->cpuid_level >= 0x0000000F) { - struct rh_cpuinfo_x86 *rh_c = - (c == &boot_cpu_data) ? &rh_boot_cpu_data - : &rh_cpu_data(c->cpu_index); - - /* QoS sub-leaf, EAX=0Fh, ECX=0 */ - cpuid_count(0x0000000F, 0, &eax, &ebx, &ecx, &edx); - c->x86_capability[CPUID_F_0_EDX] = edx; - - if (cpu_has(c, X86_FEATURE_CQM_LLC)) { - /* will be overridden if occupancy monitoring exists */ - rh_c->x86_cache_max_rmid = ebx; - - /* QoS sub-leaf, EAX=0Fh, ECX=1 */ - cpuid_count(0x0000000F, 1, &eax, &ebx, &ecx, &edx); - c->x86_capability[CPUID_F_1_EDX] = edx; - - if ((cpu_has(c, X86_FEATURE_CQM_OCCUP_LLC)) || - ((cpu_has(c, X86_FEATURE_CQM_MBM_TOTAL)) || - (cpu_has(c, X86_FEATURE_CQM_MBM_LOCAL)))) { - rh_c->x86_cache_max_rmid = ecx; - rh_c->x86_cache_occ_scale = ebx; - } - } else { - rh_c->x86_cache_max_rmid = -1; - rh_c->x86_cache_occ_scale = -1; - } - } - /* AMD-defined flags: level 0x80000001 */ eax = cpuid_eax(0x80000000); c->extended_cpuid_level = eax; @@ -883,6 +881,7 @@ void get_cpu_cap(struct cpuinfo_x86 *c) init_scattered_cpuid_features(c); init_speculation_control(c); + init_cqm(c); /* * Clear/Set all flags overridden by options, after probe. @@ -988,41 +987,49 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { {} }; +#define VULNBL(vendor, family, model, blacklist) \ + X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, blacklist) + #define VULNBL_INTEL_STEPPINGS(model, steppings, issues) \ X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE(INTEL, 6, \ INTEL_FAM6_##model, steppings, \ X86_FEATURE_ANY, issues) +#define VULNBL_AMD(family, blacklist) \ + VULNBL(AMD, family, X86_MODEL_ANY, blacklist) + #define SRBDS BIT(0) /* CPU is affected by X86_BUG_MMIO_STALE_DATA */ #define MMIO BIT(1) /* CPU is affected by Shared Buffers Data Sampling (SBDS), a variant of X86_BUG_MMIO_STALE_DATA */ #define MMIO_SBDS BIT(2) +/* CPU is affected by RETbleed, speculating where you would not expect it */ +#define RETBLEED BIT(3) static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { VULNBL_INTEL_STEPPINGS(IVYBRIDGE, X86_STEPPING_ANY, SRBDS), VULNBL_INTEL_STEPPINGS(HASWELL_CORE, X86_STEPPING_ANY, SRBDS), VULNBL_INTEL_STEPPINGS(HASWELL_ULT, X86_STEPPING_ANY, SRBDS), VULNBL_INTEL_STEPPINGS(HASWELL_GT3E, X86_STEPPING_ANY, SRBDS), - VULNBL_INTEL_STEPPINGS(HASWELL_X, BIT(2) | BIT(4), MMIO), - VULNBL_INTEL_STEPPINGS(BROADWELL_XEON_D, X86_STEPPINGS(0x3, 0x5), MMIO), + VULNBL_INTEL_STEPPINGS(HASWELL_X, X86_STEPPING_ANY, MMIO), + VULNBL_INTEL_STEPPINGS(BROADWELL_XEON_D, X86_STEPPING_ANY, MMIO), VULNBL_INTEL_STEPPINGS(BROADWELL_GT3E, X86_STEPPING_ANY, SRBDS), VULNBL_INTEL_STEPPINGS(BROADWELL_X, X86_STEPPING_ANY, MMIO), VULNBL_INTEL_STEPPINGS(BROADWELL_CORE, X86_STEPPING_ANY, SRBDS), - VULNBL_INTEL_STEPPINGS(SKYLAKE_MOBILE, X86_STEPPINGS(0x3, 0x3), SRBDS | MMIO), - VULNBL_INTEL_STEPPINGS(SKYLAKE_MOBILE, X86_STEPPING_ANY, SRBDS), - VULNBL_INTEL_STEPPINGS(SKYLAKE_X, BIT(3) | BIT(4) | BIT(6) | - BIT(7) | BIT(0xB), MMIO), - VULNBL_INTEL_STEPPINGS(SKYLAKE_DESKTOP, X86_STEPPINGS(0x3, 0x3), SRBDS | MMIO), - VULNBL_INTEL_STEPPINGS(SKYLAKE_DESKTOP, X86_STEPPING_ANY, SRBDS), - VULNBL_INTEL_STEPPINGS(KABYLAKE_MOBILE, X86_STEPPINGS(0x9, 0xC), SRBDS | MMIO), - VULNBL_INTEL_STEPPINGS(KABYLAKE_MOBILE, X86_STEPPINGS(0x0, 0x8), SRBDS), - VULNBL_INTEL_STEPPINGS(KABYLAKE_DESKTOP, X86_STEPPINGS(0x9, 0xD), SRBDS | MMIO), - VULNBL_INTEL_STEPPINGS(KABYLAKE_DESKTOP, X86_STEPPINGS(0x0, 0x8), SRBDS), - VULNBL_INTEL_STEPPINGS(ICELAKE_MOBILE, X86_STEPPINGS(0x5, 0x5), MMIO | MMIO_SBDS), - VULNBL_INTEL_STEPPINGS(ICELAKE_DESKTOP, X86_STEPPINGS(0x1, 0x1), MMIO), - VULNBL_INTEL_STEPPINGS(ICELAKE_X, X86_STEPPINGS(0x4, 0x6), MMIO), + VULNBL_INTEL_STEPPINGS(SKYLAKE_MOBILE, X86_STEPPING_ANY, SRBDS | MMIO | RETBLEED), + VULNBL_INTEL_STEPPINGS(SKYLAKE_X, X86_STEPPING_ANY, MMIO | RETBLEED), + VULNBL_INTEL_STEPPINGS(SKYLAKE_DESKTOP, X86_STEPPING_ANY, SRBDS | MMIO | RETBLEED), + VULNBL_INTEL_STEPPINGS(KABYLAKE_MOBILE, X86_STEPPING_ANY, SRBDS | MMIO | RETBLEED), + VULNBL_INTEL_STEPPINGS(KABYLAKE_DESKTOP, X86_STEPPING_ANY, SRBDS | MMIO | RETBLEED), + VULNBL_INTEL_STEPPINGS(CANNONLAKE_MOBILE, X86_STEPPING_ANY, RETBLEED), + VULNBL_INTEL_STEPPINGS(ICELAKE_MOBILE, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED), + VULNBL_INTEL_STEPPINGS(ICELAKE_DESKTOP, X86_STEPPING_ANY, MMIO), + VULNBL_INTEL_STEPPINGS(ICELAKE_X, X86_STEPPING_ANY, MMIO), VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_X, X86_STEPPING_ANY, MMIO), + + VULNBL_AMD(0x15, RETBLEED), + VULNBL_AMD(0x16, RETBLEED), + VULNBL_AMD(0x17, RETBLEED), {} }; @@ -1122,6 +1129,11 @@ static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) !arch_cap_mmio_immune(ia32_cap)) setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA); + if (!cpu_has(c, X86_FEATURE_BTC_NO)) { + if (cpu_matches(cpu_vuln_blacklist, RETBLEED) || (ia32_cap & ARCH_CAP_RSBA)) + setup_force_cpu_bug(X86_BUG_RETBLEED); + } + if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN)) return; diff --git a/arch/x86/kernel/cpu/cpuid-deps.c b/arch/x86/kernel/cpu/cpuid-deps.c index c21f22d836ad74..68adcc4af01014 100644 --- a/arch/x86/kernel/cpu/cpuid-deps.c +++ b/arch/x86/kernel/cpu/cpuid-deps.c @@ -59,6 +59,9 @@ const static struct cpuid_dep cpuid_deps[] = { { X86_FEATURE_AVX512_4VNNIW, X86_FEATURE_AVX512F }, { X86_FEATURE_AVX512_4FMAPS, X86_FEATURE_AVX512F }, { X86_FEATURE_AVX512_VPOPCNTDQ, X86_FEATURE_AVX512F }, + { X86_FEATURE_CQM_OCCUP_LLC, X86_FEATURE_CQM_LLC }, + { X86_FEATURE_CQM_MBM_TOTAL, X86_FEATURE_CQM_LLC }, + { X86_FEATURE_CQM_MBM_LOCAL, X86_FEATURE_CQM_LLC }, {} }; diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 907fb85a842ff8..45a69b1477dde3 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -21,7 +21,11 @@ struct cpuid_bit { static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_APERFMPERF, CPUID_ECX, 0, 0x00000006, 0 }, { X86_FEATURE_EPB, CPUID_ECX, 3, 0x00000006, 0 }, - { X86_FEATURE_INTEL_PT, CPUID_EBX,25, 0x00000007, 0 }, + { X86_FEATURE_RRSBA_CTRL, CPUID_EDX, 2, 0x00000007, 2 }, + { X86_FEATURE_CQM_LLC, CPUID_EDX, 1, 0x0000000f, 0 }, + { X86_FEATURE_CQM_OCCUP_LLC, CPUID_EDX, 0, 0x0000000f, 1 }, + { X86_FEATURE_CQM_MBM_TOTAL, CPUID_EDX, 1, 0x0000000f, 1 }, + { X86_FEATURE_CQM_MBM_LOCAL, CPUID_EDX, 2, 0x0000000f, 1 }, { X86_FEATURE_CAT_L3, CPUID_EBX, 1, 0x00000010, 0 }, { X86_FEATURE_CAT_L2, CPUID_EBX, 2, 0x00000010, 0 }, { X86_FEATURE_CDP_L3, CPUID_ECX, 2, 0x00000010, 1 }, diff --git a/arch/x86/kernel/entry.S b/arch/x86/kernel/entry.S new file mode 100644 index 00000000000000..9b0128706892e3 --- /dev/null +++ b/arch/x86/kernel/entry.S @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common place for both 32- and 64-bit entry routines. + */ + +#include +#include + +ENTRY(entry_ibpb) + movl $MSR_IA32_PRED_CMD, %ecx + movl $PRED_CMD_IBPB, %eax + xorl %edx, %edx + wrmsr + RET +ENDPROC(entry_ibpb) diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S index 47cae28d9a1297..60d4f730e972e4 100644 --- a/arch/x86/kernel/entry_32.S +++ b/arch/x86/kernel/entry_32.S @@ -1087,7 +1087,7 @@ BUILD_INTERRUPT3(hv_stimer0_callback_vector, HYPERV_STIMER0_VECTOR, #ifdef CONFIG_DYNAMIC_FTRACE ENTRY(mcount) - ret + RET END(mcount) ENTRY(ftrace_caller) @@ -1117,7 +1117,7 @@ ftrace_graph_call: .globl ftrace_stub ftrace_stub: - ret + RET END(ftrace_caller) ENTRY(ftrace_regs_caller) @@ -1196,7 +1196,7 @@ ENTRY(mcount) #endif .globl ftrace_stub ftrace_stub: - ret + RET /* taken from glibc */ trace: @@ -1230,7 +1230,7 @@ ENTRY(ftrace_graph_caller) popl %edx popl %ecx popl %eax - ret + RET END(ftrace_graph_caller) .globl return_to_handler diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 21508db7edd6aa..61073a219beaab 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -269,7 +269,8 @@ ENDPROC(native_usergs_sysret64) SWITCH_TO_KERNEL_CR3 IBRS_ENTRY_CLOBBER /* no indirect jump allowed before IBRS */ movq PER_CPU_VAR(kernel_stack), %rsp - FILL_RETURN_BUFFER_CLOBBER /* no ret allowed before stuffing the RSB */ + FILL_RETURN_BUFFER_CLOBBER %rax, X86_FEATURE_RSB_CTXSW /* no ret allowed before stuffing the RSB */ + UNTRAIN_RET movq %rsi, %rsp jmp 2f 1: @@ -329,7 +330,7 @@ ENTRY(save_rest) movq_cfi r15, R15+16 movq %r11, 8(%rsp) /* return address */ FIXUP_TOP_OF_STACK %r11, 16 - ret + RET CFI_ENDPROC END(save_rest) @@ -367,8 +368,9 @@ ENTRY(save_paranoid) 2: SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14 IBRS_ENTRY_SAVE_AND_CLOBBER save_reg=%r13d /* no indirect jump allowed before IBRS */ - FILL_RETURN_BUFFER_CLOBBER /* no ret allowed before stuffing the RSB */ - ret + FILL_RETURN_BUFFER_CLOBBER %rax, X86_FEATURE_RSB_CTXSW /* no ret allowed before stuffing the RSB */ + UNTRAIN_RET + RET CFI_ENDPROC END(save_paranoid) .popsection @@ -475,7 +477,8 @@ GLOBAL(system_call_after_swapgs) movq %rsp,PER_CPU_VAR(old_rsp) movq PER_CPU_VAR(kernel_stack),%rsp IBRS_ENTRY /* no indirect jump allowed before IBRS */ - FILL_RETURN_BUFFER /* no ret allowed before stuffing the RSB */ + FILL_RETURN_BUFFER %rax, X86_FEATURE_RSB_CTXSW /* no ret allowed before stuffing the RSB */ + UNTRAIN_RET /* * No need to follow this irqs off/on section - it's straight * and short: @@ -715,7 +718,7 @@ ENTRY(stub_\func) call sys_\func UNWIND_END_OF_STACK RESTORE_TOP_OF_STACK %r11, 8 - ret + RET CFI_ENDPROC END(stub_\func) .endm @@ -728,7 +731,7 @@ ENTRY(\label) call \func UNWIND_END_OF_STACK RESTORE_TOP_OF_STACK %r11, 8 - ret + RET CFI_ENDPROC END(\label) .endm @@ -1269,7 +1272,7 @@ gs_change: 2: mfence /* workaround */ SWAPGS popfq_cfi - ret + RET CFI_ENDPROC END(native_load_gs_index) @@ -1299,7 +1302,7 @@ ENTRY(call_softirq) CFI_DEF_CFA_REGISTER rsp CFI_ADJUST_CFA_OFFSET -8 decl PER_CPU_VAR(irq_count) - ret + RET CFI_ENDPROC END(call_softirq) @@ -1541,7 +1544,8 @@ error_swapgs: movq %rsp, %rsi movq PER_CPU_VAR(kernel_stack), %rsp IBRS_ENTRY_CLOBBER /* no indirect jump allowed before IBRS */ - FILL_RETURN_BUFFER_CLOBBER /* no ret allowed before stuffing the RSB */ + FILL_RETURN_BUFFER_CLOBBER %rax, X86_FEATURE_RSB_CTXSW /* no ret allowed before stuffing the RSB */ + UNTRAIN_RET movq %rsi, %rsp error_sti: /* Check to see if we're on the trampoline stack. */ @@ -1565,7 +1569,7 @@ error_sti: movq %rax, %rsp 1: TRACE_IRQS_OFF - ret + RET /* * There are two places in the kernel that can potentially fault with @@ -1727,7 +1731,8 @@ ENTRY(nmi) pushq %r15 /* pt_regs->r15 */ IBRS_ENTRY_CLOBBER /* no indirect jump allowed before IBRS */ - FILL_RETURN_BUFFER_CLOBBER /* no ret allowed before stuffing the RSB */ + FILL_RETURN_BUFFER_CLOBBER %rax, X86_FEATURE_RSB_CTXSW /* no ret allowed before stuffing the RSB */ + UNTRAIN_RET /* * At this point we no longer need to worry about stack damage diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S index 24f28c480484d5..1ccfe8d40d48ff 100644 --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S @@ -476,11 +476,11 @@ check_x87: movl %cr0,%eax /* no coprocessor: have to set bits */ xorl $4,%eax /* set EM */ movl %eax,%cr0 - ret + RET ALIGN 1: movb $1,X86_HARD_MATH .byte 0xDB,0xE4 /* fsetpm for 287, ignored by 387 */ - ret + RET #include "verify_cpu.S" @@ -540,7 +540,7 @@ setup_once: #endif andl $0,setup_once_ref /* Once is enough, thanks */ - ret + RET ENTRY(early_idt_handler_array) # 36(%esp) %eflags diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index c814940824742d..6adc9fbcb920c2 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -672,7 +672,7 @@ static void __used __kprobes kretprobe_trampoline_holder(void) RESTORE_REGS_STRING " popf\n" #endif - " ret\n"); + ASM_RET); } STACK_FRAME_NON_STANDARD(kretprobe_trampoline_holder); STACK_FRAME_NON_STANDARD(kretprobe_trampoline); diff --git a/arch/x86/kernel/kprobes/ftrace.c b/arch/x86/kernel/kprobes/ftrace.c index 1b7a6df9451115..ef22fcdd7d2da8 100644 --- a/arch/x86/kernel/kprobes/ftrace.c +++ b/arch/x86/kernel/kprobes/ftrace.c @@ -99,7 +99,7 @@ asmlinkage void override_func(void); asm( ".type override_func, @function\n" "override_func:\n" - " ret\n" + ASM_RET ".size override_func, .-override_func\n" ); diff --git a/arch/x86/kernel/mcount_64.S b/arch/x86/kernel/mcount_64.S index daeb099cf95e61..911e9974ac9571 100644 --- a/arch/x86/kernel/mcount_64.S +++ b/arch/x86/kernel/mcount_64.S @@ -24,7 +24,7 @@ #ifdef CONFIG_DYNAMIC_FTRACE ENTRY(function_hook) - retq + RET END(function_hook) /* skip is set if stack has been adjusted */ @@ -63,7 +63,7 @@ GLOBAL(ftrace_graph_call) /* This is weak to keep gas from relaxing the jumps */ WEAK(ftrace_stub) - retq + RET END(ftrace_caller) ENTRY(ftrace_regs_caller) @@ -146,7 +146,7 @@ ENTRY(function_hook) #endif GLOBAL(ftrace_stub) - retq + RET trace: MCOUNT_SAVE_FRAME @@ -187,7 +187,7 @@ ENTRY(ftrace_graph_caller) MCOUNT_RESTORE_FRAME - retq + RET END(ftrace_graph_caller) GLOBAL(return_to_handler) diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index e93a3a4b53c8b2..151d07fa4c8172 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -249,7 +249,7 @@ int module_finalize(const Elf_Ehdr *hdr, struct module *me) { const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL, - *para = NULL; + *para = NULL, *returns = NULL; char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { @@ -261,8 +261,14 @@ int module_finalize(const Elf_Ehdr *hdr, locks = s; if (!strcmp(".parainstructions", secstrings + s->sh_name)) para = s; + if (!strcmp(".return_sites", secstrings + s->sh_name)) + returns = s; } + if (returns) { + void *rseg = (void *)returns->sh_addr; + apply_returns(rseg, rseg + returns->sh_size); + } if (alt) { /* patch .altinstructions */ void *aseg = (void *)alt->sh_addr; diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index da6c7e5ee57d4f..7629803fe38807 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -48,7 +48,7 @@ extern void _paravirt_nop(void); asm (".pushsection .entry.text, \"ax\"\n" ".global _paravirt_nop\n" "_paravirt_nop:\n\t" - "ret\n\t" + ASM_RET ".size _paravirt_nop, . - _paravirt_nop\n\t" ".type _paravirt_nop, @function\n\t" ".popsection"); diff --git a/arch/x86/kernel/relocate_kernel_32.S b/arch/x86/kernel/relocate_kernel_32.S index 36818f8ec2be08..e2f2b8fad40ee3 100644 --- a/arch/x86/kernel/relocate_kernel_32.S +++ b/arch/x86/kernel/relocate_kernel_32.S @@ -12,7 +12,8 @@ #include /* - * Must be relocatable PIC code callable as a C function + * Must be relocatable PIC code callable as a C function, in particular + * there must be a plain RET and not jump to return thunk. */ #define PTR(x) (x << 2) @@ -95,6 +96,7 @@ relocate_kernel: addl $(identity_mapped - relocate_kernel), %eax pushl %eax ret + int3 identity_mapped: /* set return address to 0 if not preserving context */ @@ -162,6 +164,7 @@ identity_mapped: xorl %esi, %esi xorl %ebp, %ebp ret + int3 1: popl %edx movl CP_PA_SWAP_PAGE(%edi), %esp @@ -193,6 +196,7 @@ identity_mapped: addl $(virtual_mapped - relocate_kernel), %eax pushl %eax ret + int3 virtual_mapped: movl CR4(%edi), %eax @@ -210,6 +214,7 @@ virtual_mapped: popl %esi popl %ebx ret + int3 /* Do the copies */ swap_pages: @@ -272,6 +277,7 @@ swap_pages: popl %ebx popl %ebp ret + int3 .globl kexec_control_code_size .set kexec_control_code_size, . - relocate_kernel diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index a59dd47921a27b..191df3d64a1fca 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -13,7 +13,8 @@ #include /* - * Must be relocatable PIC code callable as a C function + * Must be relocatable PIC code callable as a C function, in particular + * there must be a plain RET and not jump to return thunk. */ #define PTR(x) (x << 3) @@ -102,6 +103,7 @@ relocate_kernel: addq $(identity_mapped - relocate_kernel), %r8 pushq %r8 ret + int3 identity_mapped: /* set return address to 0 if not preserving context */ @@ -182,6 +184,7 @@ identity_mapped: xorq %r15, %r15 ret + int3 1: popq %rdx @@ -203,6 +206,7 @@ identity_mapped: movq $virtual_mapped, %rax pushq %rax ret + int3 virtual_mapped: movq RSP(%r8), %rsp @@ -222,6 +226,7 @@ virtual_mapped: popq %rbp popq %rbx ret + int3 /* Do the copies */ swap_pages: @@ -277,6 +282,7 @@ swap_pages: jmp 0b 3: ret + int3 .globl kexec_control_code_size .set kexec_control_code_size, . - relocate_kernel diff --git a/arch/x86/kernel/spec_ctrl.c b/arch/x86/kernel/spec_ctrl.c index 7601d0b59dc7c7..bd9153f54b0f0f 100644 --- a/arch/x86/kernel/spec_ctrl.c +++ b/arch/x86/kernel/spec_ctrl.c @@ -423,8 +423,11 @@ bool spec_ctrl_force_enable_ibrs(void) bool spec_ctrl_cond_enable_ibrs(bool full_retp) { - if (cpu_has_spec_ctrl() && (is_skylake_era() || !full_retp) && - !noibrs_cmdline) { + bool retbleed = boot_cpu_has_bug(X86_BUG_RETBLEED) && + boot_cpu_data.x86_vendor == X86_VENDOR_INTEL; + + if (cpu_has_spec_ctrl() && + (retbleed || is_skylake_era() || !full_retp) && !noibrs_cmdline) { if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { spec_ctrl_enable_ibrs_enhanced(); } else { @@ -868,6 +871,9 @@ static ssize_t retp_enabled_read(struct file *file, char __user *user_buf, return __enabled_read(file, user_buf, count, ppos, &enabled); } +#define RETP_IBRS_OFF_WARN "WARNING: disabling IBRS on a RETBleed affected CPU is not safe." +#define RETP_IBRS_ON_FAIL "WARNING: failed to set IBRS. CPU vulnerable to RETBleed attacks." + static ssize_t retp_enabled_write(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) @@ -884,17 +890,47 @@ static ssize_t retp_enabled_write(struct file *file, if (kstrtouint(buf, 0, &enable)) return -EINVAL; - if (enable > 1) + if (enable > 2) return -EINVAL; mutex_lock(&spec_ctrl_mutex); - if (enable == retp_enabled()) + if (!!enable == retp_enabled()) goto out_unlock; - set_spec_ctrl_retp(enable); - if (enable) { + bool force_enable = (enable == 2) ? true : false; + + /* + * RETBleed affected CPUs (Intel) depend on IBRS as + * an effective mitigation mechanism. We'll override + * retp_enabled=1 with spectre_v2=ibrs here. + */ + if (boot_cpu_has_bug(X86_BUG_RETBLEED) && + boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { + if (!force_enable) { + if (ibrs_mode == IBRS_DISABLED) { + pr_warn("%s Switching to spectre_v2=ibrs\n", + RETP_IBRS_OFF_WARN); + + if (spec_ctrl_cond_enable_ibrs(true)) + goto out_get_mitigation; + else + pr_warn("%s\n", RETP_IBRS_ON_FAIL); + } + + goto out_unlock; + } + + /* + * retp_enabled=2 case here, which preserves the + * old behavior and disables IBRS. + * on a RETBleed affected CPU we just throw the + * warning and keep on going. + */ + pr_warn("%s\n", RETP_IBRS_OFF_WARN); + } + /* enforce sane combinations */ if (ibp_disabled) { sync_all_cpus_ibp(true); @@ -907,6 +943,10 @@ static ssize_t retp_enabled_write(struct file *file, set_spec_ctrl_pcp_ibrs_user(); } } + + set_spec_ctrl_retp(!!enable); + +out_get_mitigation: spec_ctrl_get_mitigation(); out_unlock: diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index dad7e081f43d5e..c9b139ae96803c 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -225,6 +225,15 @@ SECTIONS __parainstructions_end = .; } +#ifdef CONFIG_RETPOLINE + . = ALIGN(8); + .return_sites : AT(ADDR(.return_sites) - LOAD_OFFSET) { + __return_sites = .; + *(.return_sites) + __return_sites_end = .; + } +#endif + /* * struct alt_inst entries. From the header (alternative.h): * "Alternative instructions for different CPU types or capabilities" diff --git a/arch/x86/kernel/vsyscall_emu_64.S b/arch/x86/kernel/vsyscall_emu_64.S index c9596a9af15985..1edd6a47109b78 100644 --- a/arch/x86/kernel/vsyscall_emu_64.S +++ b/arch/x86/kernel/vsyscall_emu_64.S @@ -21,16 +21,19 @@ __vsyscall_page: mov $__NR_gettimeofday, %rax syscall ret + int3 .balign 1024, 0xcc mov $__NR_time, %rax syscall ret + int3 .balign 1024, 0xcc mov $__NR_getcpu, %rax syscall ret + int3 .balign 4096, 0xcc diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c index 5d094269329734..427222ddc1903b 100644 --- a/arch/x86/kernel/x8664_ksyms_64.c +++ b/arch/x86/kernel/x8664_ksyms_64.c @@ -97,4 +97,13 @@ EXPORT_THUNK(r12); EXPORT_THUNK(r13); EXPORT_THUNK(r14); EXPORT_THUNK(r15); + +extern void __x86_return_thunk(void); +EXPORT_SYMBOL(__x86_return_thunk); + +extern void zen_untrain_ret(void); +EXPORT_SYMBOL(zen_untrain_ret); + +extern void entry_ibpb(void); +EXPORT_SYMBOL_GPL(entry_ibpb); #endif diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index 355e2806e1a25a..b67e56d187483d 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -45,8 +45,6 @@ static const struct cpuid_reg reverse_cpuid[] = { [CPUID_8000_0001_ECX] = {0x80000001, 0, CPUID_ECX}, [CPUID_7_0_EBX] = { 7, 0, CPUID_EBX}, [CPUID_D_1_EAX] = { 0xd, 1, CPUID_EAX}, - [CPUID_F_0_EDX] = { 0xf, 0, CPUID_EDX}, - [CPUID_F_1_EDX] = { 0xf, 1, CPUID_EDX}, [CPUID_8000_0008_EBX] = {0x80000008, 0, CPUID_EBX}, [CPUID_6_EAX] = { 6, 0, CPUID_EAX}, [CPUID_8000_000A_EDX] = {0x8000000a, 0, CPUID_EDX}, diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 8f620c23db2a51..7ad357e04a9571 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -188,9 +188,6 @@ #define X8(x...) X4(x), X4(x) #define X16(x...) X8(x), X8(x) -#define NR_FASTOP (ilog2(sizeof(ulong)) + 1) -#define FASTOP_SIZE 8 - /* * fastop functions have a special calling convention: * @@ -204,6 +201,10 @@ * different operand sizes can be reached by calculation, rather than a jump * table (which would be bigger than the code). * + * The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR + * and 1 for the straight line speculation INT3, leaves 7 bytes for the + * body of the function. Currently none is larger than 4. + * * fastop functions are declared as taking a never-defined fastop parameter, * so they can't be called from C directly. */ @@ -332,29 +333,46 @@ static void invalidate_registers(struct x86_emulate_ctxt *ctxt) static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *)); -#define FOP_FUNC(name) \ +#define FASTOP_SIZE 16 + +#define __FOP_FUNC(name) \ ".align " __stringify(FASTOP_SIZE) " \n\t" \ ".type " name ", @function \n\t" \ name ":\n\t" -#define FOP_RET "ret \n\t" +#define FOP_FUNC(name) \ + __FOP_FUNC(#name) + +#define __FOP_RET(name) \ + ASM_RET \ + ".size " name ", .-" name "\n\t" -#define FOP_START(op) \ +#define FOP_RET(name) \ + __FOP_RET(#name) + +#define __FOP_START(op, align) \ extern void em_##op(struct fastop *fake); \ asm(".pushsection .text, \"ax\" \n\t" \ ".global em_" #op " \n\t" \ - FOP_FUNC("em_" #op) + ".align " __stringify(align) " \n\t" \ + "em_" #op ":\n\t" + +#define FOP_START(op) __FOP_START(op, FASTOP_SIZE) #define FOP_END \ ".popsection") +#define __FOPNOP(name) \ + __FOP_FUNC(name) \ + __FOP_RET(name) + #define FOPNOP() \ - FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \ - FOP_RET + __FOPNOP(__stringify(__UNIQUE_ID(nop))) #define FOP1E(op, dst) \ - FOP_FUNC(#op "_" #dst) \ - "10: " #op " %" #dst " \n\t" FOP_RET + __FOP_FUNC(#op "_" #dst) \ + "10: " #op " %" #dst " \n\t" \ + __FOP_RET(#op "_" #dst) #define FOP1EEX(op, dst) \ FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception) @@ -386,8 +404,9 @@ static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *)); FOP_END #define FOP2E(op, dst, src) \ - FOP_FUNC(#op "_" #dst "_" #src) \ - #op " %" #src ", %" #dst " \n\t" FOP_RET + __FOP_FUNC(#op "_" #dst "_" #src) \ + #op " %" #src ", %" #dst " \n\t" \ + __FOP_RET(#op "_" #dst "_" #src) #define FASTOP2(op) \ FOP_START(op) \ @@ -425,8 +444,9 @@ static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *)); FOP_END #define FOP3E(op, dst, src, src2) \ - FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \ - #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET + __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \ + #op " %" #src2 ", %" #src ", %" #dst " \n\t"\ + __FOP_RET(#op "_" #dst "_" #src "_" #src2) /* 3-operand, word-only, src2=cl */ #define FASTOP3WCL(op) \ @@ -437,18 +457,28 @@ static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *)); ON64(FOP3E(op##q, rax, rdx, cl)) \ FOP_END +/* + * Depending on .config the SETcc functions look like: + * SETcc %al [3 bytes] + * RET | JMP __x86_return_thunk [1,5 bytes; CONFIG_RETPOLINE] + */ +#define SETCC_ALIGN 16 + /* Special case for SETcc - 1 instruction per cc */ #define FOP_SETCC(op) \ ".align 4 \n\t" \ ".type " #op ", @function \n\t" \ #op ": \n\t" \ #op " %al \n\t" \ - FOP_RET + __FOP_RET(#op) \ + ".skip " __stringify(SETCC_ALIGN) " - (.-" #op "), 0xcc \n\t" -asm(".global kvm_fastop_exception \n" - "kvm_fastop_exception: xor %esi, %esi; ret"); +asm(".pushsection .fixup, \"ax\"\n" + ".global kvm_fastop_exception \n" + "kvm_fastop_exception: xor %esi, %esi; " ASM_RET + ".popsection"); -FOP_START(setcc) +__FOP_START(setcc, SETCC_ALIGN) FOP_SETCC(seto) FOP_SETCC(setno) FOP_SETCC(setc) @@ -467,7 +497,10 @@ FOP_SETCC(setle) FOP_SETCC(setnle) FOP_END; -FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET +FOP_START(salc) +FOP_FUNC(salc) +"pushf; sbb %al, %al; popf \n\t" +FOP_RET(salc) FOP_END; /* diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 7e21ec91412aa7..c7cf69eefbd33d 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -5272,6 +5272,15 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) /* Eliminate branch target predictions from guest mode */ fill_RSB(); + /* + * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be + * untrained as soon as we exit the VM and are back to the + * kernel. This should be done before re-enabling interrupts + * because interrupt handlers won't sanitize 'ret' if the return is + * from the kernel. + */ + untrain_ret(); + reload_tss(vcpu); local_irq_disable(); diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index a3d1838559a4f5..920a7dfa31cbd1 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -9569,6 +9569,15 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) /* Eliminate branch target predictions from guest mode */ fill_RSB(); + /* + * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be + * untrained as soon as we exit the VM and are back to the + * kernel. This should be done before re-enabling interrupts + * because interrupt handlers won't sanitize 'ret' if the return is + * from the kernel. + */ + untrain_ret(); + /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */ if (vmx->host_debugctlmsr) update_debugctlmsr(vmx->host_debugctlmsr); diff --git a/arch/x86/lguest/i386_head.S b/arch/x86/lguest/i386_head.S index 6ddfe4fc23c3ee..fbeffa2db38f85 100644 --- a/arch/x86/lguest/i386_head.S +++ b/arch/x86/lguest/i386_head.S @@ -91,7 +91,7 @@ ENTRY(lg_irq_enable) * a register. In this case, the normal path hasn't needed to save or * restore any registers at all! */ - ret + RET send_interrupts: /* * OK, now we need a register: eax is used for the hypercall number, @@ -109,7 +109,7 @@ send_interrupts: int $LGUEST_TRAP_ENTRY /* Put eax back the way we found it. */ popl %eax - ret + RET /* * Finally, the "popf" or "restore flags" routine. The %eax register holds the @@ -130,7 +130,7 @@ ENTRY(lg_restore_fl) testl lguest_data+LGUEST_DATA_irq_pending, %eax jnz send_interrupts /* Again, the normal path has used no extra registers. Clever, huh? */ - ret + RET /*:*/ /* These demark the EIP range where host should never deliver interrupts. */ diff --git a/arch/x86/lib/atomic64_386_32.S b/arch/x86/lib/atomic64_386_32.S index 00933d5e992f7a..a473ab5872bf2b 100644 --- a/arch/x86/lib/atomic64_386_32.S +++ b/arch/x86/lib/atomic64_386_32.S @@ -37,7 +37,7 @@ ENTRY(atomic64_##op##_386); \ #define RET \ UNLOCK v; \ - ret + RET #define RET_ENDP \ RET; \ diff --git a/arch/x86/lib/atomic64_cx8_32.S b/arch/x86/lib/atomic64_cx8_32.S index f5cc9eb1d51bc0..5ed0129ac92df7 100644 --- a/arch/x86/lib/atomic64_cx8_32.S +++ b/arch/x86/lib/atomic64_cx8_32.S @@ -35,7 +35,7 @@ ENTRY(atomic64_read_cx8) CFI_STARTPROC read64 %ecx - ret + RET CFI_ENDPROC ENDPROC(atomic64_read_cx8) @@ -48,7 +48,7 @@ ENTRY(atomic64_set_cx8) cmpxchg8b (%esi) jne 1b - ret + RET CFI_ENDPROC ENDPROC(atomic64_set_cx8) @@ -60,7 +60,7 @@ ENTRY(atomic64_xchg_cx8) cmpxchg8b (%esi) jne 1b - ret + RET CFI_ENDPROC ENDPROC(atomic64_xchg_cx8) @@ -93,7 +93,7 @@ ENTRY(atomic64_\func\()_return_cx8) RESTORE esi RESTORE ebx RESTORE ebp - ret + RET CFI_ENDPROC ENDPROC(atomic64_\func\()_return_cx8) .endm @@ -120,7 +120,7 @@ ENTRY(atomic64_\func\()_return_cx8) movl %ebx, %eax movl %ecx, %edx RESTORE ebx - ret + RET CFI_ENDPROC ENDPROC(atomic64_\func\()_return_cx8) .endm @@ -147,7 +147,7 @@ ENTRY(atomic64_dec_if_positive_cx8) movl %ebx, %eax movl %ecx, %edx RESTORE ebx - ret + RET CFI_ENDPROC ENDPROC(atomic64_dec_if_positive_cx8) @@ -181,7 +181,7 @@ ENTRY(atomic64_add_unless_cx8) CFI_ADJUST_CFA_OFFSET -8 RESTORE ebx RESTORE ebp - ret + RET 4: cmpl %edx, 4(%esp) jne 2b @@ -210,6 +210,6 @@ ENTRY(atomic64_inc_not_zero_cx8) movl $1, %eax 3: RESTORE ebx - ret + RET CFI_ENDPROC ENDPROC(atomic64_inc_not_zero_cx8) diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S index e78b8eee66155d..bbfa68fd1f9707 100644 --- a/arch/x86/lib/checksum_32.S +++ b/arch/x86/lib/checksum_32.S @@ -135,7 +135,7 @@ ENTRY(csum_partial) CFI_RESTORE ebx popl_cfi %esi CFI_RESTORE esi - ret + RET CFI_ENDPROC ENDPROC(csum_partial) @@ -259,7 +259,7 @@ ENTRY(csum_partial) CFI_RESTORE ebx popl_cfi %esi CFI_RESTORE esi - ret + RET CFI_ENDPROC ENDPROC(csum_partial) @@ -419,7 +419,7 @@ DST( movb %cl, (%edi) ) popl_cfi %edi CFI_RESTORE edi popl_cfi %ecx # equivalent to addl $4,%esp - ret + RET CFI_ENDPROC ENDPROC(csum_partial_copy_generic) @@ -512,7 +512,7 @@ DST( movb %dl, (%edi) ) CFI_RESTORE edi popl_cfi %ebx CFI_RESTORE ebx - ret + RET CFI_ENDPROC ENDPROC(csum_partial_copy_generic) diff --git a/arch/x86/lib/clear_page_64.S b/arch/x86/lib/clear_page_64.S index 82767a28144adc..73a49dae358ec2 100644 --- a/arch/x86/lib/clear_page_64.S +++ b/arch/x86/lib/clear_page_64.S @@ -11,7 +11,7 @@ ENTRY(clear_page_c) movl $4096/8,%ecx xorl %eax,%eax rep stosq - ret + RET CFI_ENDPROC ENDPROC(clear_page_c) @@ -20,7 +20,7 @@ ENTRY(clear_page_c_e) movl $4096,%ecx xorl %eax,%eax rep stosb - ret + RET CFI_ENDPROC ENDPROC(clear_page_c_e) @@ -43,7 +43,7 @@ ENTRY(clear_page) leaq 64(%rdi),%rdi jnz .Lloop nop - ret + RET CFI_ENDPROC .Lclear_page_end: ENDPROC(clear_page) diff --git a/arch/x86/lib/cmpxchg16b_emu.S b/arch/x86/lib/cmpxchg16b_emu.S index 1e572c507d06c2..fbed80fa80b30c 100644 --- a/arch/x86/lib/cmpxchg16b_emu.S +++ b/arch/x86/lib/cmpxchg16b_emu.S @@ -53,12 +53,12 @@ this_cpu_cmpxchg16b_emu: popf mov $1, %al - ret + RET not_same: popf xor %al,%al - ret + RET CFI_ENDPROC diff --git a/arch/x86/lib/cmpxchg8b_emu.S b/arch/x86/lib/cmpxchg8b_emu.S index 828cb710dec29c..7de8a11785d2d0 100644 --- a/arch/x86/lib/cmpxchg8b_emu.S +++ b/arch/x86/lib/cmpxchg8b_emu.S @@ -43,7 +43,7 @@ cmpxchg8b_emu: movl %ecx, 4(%esi) popfl - ret + RET not_same: movl (%esi), %eax @@ -51,7 +51,7 @@ cmpxchg8b_emu: movl 4(%esi), %edx popfl - ret + RET CFI_ENDPROC ENDPROC(cmpxchg8b_emu) diff --git a/arch/x86/lib/copy_page_64.S b/arch/x86/lib/copy_page_64.S index c6acbc3db5afef..029140d01f1064 100644 --- a/arch/x86/lib/copy_page_64.S +++ b/arch/x86/lib/copy_page_64.S @@ -9,7 +9,7 @@ copy_page_rep: CFI_STARTPROC movl $4096/8, %ecx rep movsq - ret + RET CFI_ENDPROC ENDPROC(copy_page_rep) @@ -89,7 +89,7 @@ ENTRY(copy_page) CFI_RESTORE r12 addq $2*8, %rsp CFI_ADJUST_CFA_OFFSET -2*8 - ret + RET .Lcopy_page_end: CFI_ENDPROC ENDPROC(copy_page) diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S index 8725b418fdfa56..d8860a16f83ac9 100644 --- a/arch/x86/lib/copy_user_64.S +++ b/arch/x86/lib/copy_user_64.S @@ -111,7 +111,7 @@ bad_from_user: stosb bad_to_user: movl %edx,%eax - ret + RET CFI_ENDPROC ENDPROC(bad_from_user) .previous @@ -180,7 +180,7 @@ ENTRY(copy_user_generic_unrolled) jnz 21b 23: xor %eax,%eax ASM_CLAC - ret + RET .section .fixup,"ax" 30: shll $6,%ecx @@ -251,7 +251,7 @@ ENTRY(copy_user_generic_string) movsb 4: xorl %eax,%eax ASM_CLAC - ret + RET .section .fixup,"ax" 11: lea (%rdx,%rcx,8),%rcx @@ -286,7 +286,7 @@ ENTRY(copy_user_enhanced_fast_string) movsb 2: xorl %eax,%eax ASM_CLAC - ret + RET .section .fixup,"ax" 12: movl %ecx,%edx /* ecx is zerorest also */ diff --git a/arch/x86/lib/copy_user_nocache_64.S b/arch/x86/lib/copy_user_nocache_64.S index 75d4c74ace6153..4267b6356f0075 100644 --- a/arch/x86/lib/copy_user_nocache_64.S +++ b/arch/x86/lib/copy_user_nocache_64.S @@ -150,7 +150,7 @@ ENTRY(__copy_user_nocache) xorl %eax,%eax ASM_CLAC sfence - ret + RET .section .fixup,"ax" .L_fixup_4x8b_copy: diff --git a/arch/x86/lib/csum-copy_64.S b/arch/x86/lib/csum-copy_64.S index 2419d5fefae30a..6a7b9d440b8446 100644 --- a/arch/x86/lib/csum-copy_64.S +++ b/arch/x86/lib/csum-copy_64.S @@ -220,7 +220,7 @@ ENTRY(csum_partial_copy_generic) CFI_RESTORE rbp addq $7*8, %rsp CFI_ADJUST_CFA_OFFSET -7*8 - ret + RET CFI_RESTORE_STATE /* Exception handlers. Very simple, zeroing is done in the wrappers */ diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S index 3917307fca99f2..ef2819da50520c 100644 --- a/arch/x86/lib/getuser.S +++ b/arch/x86/lib/getuser.S @@ -46,7 +46,7 @@ ENTRY(__get_user_1) 1: movzbl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET CFI_ENDPROC ENDPROC(__get_user_1) @@ -63,7 +63,7 @@ ENTRY(__get_user_2) 2: movzwl -1(%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET CFI_ENDPROC ENDPROC(__get_user_2) @@ -80,7 +80,7 @@ ENTRY(__get_user_4) 3: movl -3(%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET CFI_ENDPROC ENDPROC(__get_user_4) @@ -98,7 +98,7 @@ ENTRY(__get_user_8) 4: movq -7(%_ASM_AX),%rdx xor %eax,%eax ASM_CLAC - ret + RET #else add $7,%_ASM_AX jc bad_get_user_8 @@ -112,7 +112,7 @@ ENTRY(__get_user_8) 5: movl -3(%_ASM_AX),%ecx xor %eax,%eax ASM_CLAC - ret + RET #endif CFI_ENDPROC ENDPROC(__get_user_8) @@ -123,7 +123,7 @@ bad_get_user: xor %edx,%edx mov $(-EFAULT),%_ASM_AX ASM_CLAC - ret + RET CFI_ENDPROC END(bad_get_user) @@ -134,7 +134,7 @@ bad_get_user_8: xor %ecx,%ecx mov $(-EFAULT),%_ASM_AX ASM_CLAC - ret + RET CFI_ENDPROC END(bad_get_user_8) #endif diff --git a/arch/x86/lib/iomap_copy_64.S b/arch/x86/lib/iomap_copy_64.S index 05a95e713da885..6e13c8ab45a6ec 100644 --- a/arch/x86/lib/iomap_copy_64.S +++ b/arch/x86/lib/iomap_copy_64.S @@ -25,6 +25,6 @@ ENTRY(__iowrite32_copy) CFI_STARTPROC movl %edx,%ecx rep movsd - ret + RET CFI_ENDPROC ENDPROC(__iowrite32_copy) diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S index dcf2dd48a12293..bb72b735ce36f4 100644 --- a/arch/x86/lib/memcpy_64.S +++ b/arch/x86/lib/memcpy_64.S @@ -137,7 +137,7 @@ ENTRY(memcpy) movq %r9, 1*8(%rdi) movq %r10, -2*8(%rdi, %rdx) movq %r11, -1*8(%rdi, %rdx) - retq + RET .p2align 4 .Lless_16bytes: cmpl $8, %edx @@ -149,7 +149,7 @@ ENTRY(memcpy) movq -1*8(%rsi, %rdx), %r9 movq %r8, 0*8(%rdi) movq %r9, -1*8(%rdi, %rdx) - retq + RET .p2align 4 .Lless_8bytes: cmpl $4, %edx @@ -162,7 +162,7 @@ ENTRY(memcpy) movl -4(%rsi, %rdx), %r8d movl %ecx, (%rdi) movl %r8d, -4(%rdi, %rdx) - retq + RET .p2align 4 .Lless_3bytes: subl $1, %edx @@ -180,7 +180,7 @@ ENTRY(memcpy) movb %cl, (%rdi) .Lend: - retq + RET CFI_ENDPROC ENDPROC(memcpy) ENDPROC(__memcpy) @@ -280,7 +280,7 @@ ENTRY(__memcpy_mcsafe) /* Copy successful. Return zero */ .L_done_memcpy_trap: xorq %rax, %rax - ret + RET ENDPROC(__memcpy_mcsafe) .section .fixup, "ax" @@ -295,7 +295,7 @@ ENDPROC(__memcpy_mcsafe) addl %edx, %ecx .E_trailing_bytes: mov %ecx, %eax - ret + RET /* * For write fault handling, given the destination is unaligned, diff --git a/arch/x86/lib/memmove_64.S b/arch/x86/lib/memmove_64.S index 284f9d71e15bd5..88f94f884346eb 100644 --- a/arch/x86/lib/memmove_64.S +++ b/arch/x86/lib/memmove_64.S @@ -202,7 +202,7 @@ ENTRY(memmove) movb (%rsi), %r11b movb %r11b, (%rdi) 13: - retq + RET CFI_ENDPROC .section .altinstr_replacement,"ax" diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S index 14df4d77b69ab9..aaa43c9bf6696e 100644 --- a/arch/x86/lib/memset_64.S +++ b/arch/x86/lib/memset_64.S @@ -118,7 +118,7 @@ ENTRY(__memset) .Lende: movq %r10,%rax - ret + RET CFI_RESTORE_STATE .Lbad_alignment: diff --git a/arch/x86/lib/msr-reg.S b/arch/x86/lib/msr-reg.S index f6d13eefad1063..ce5558dfb40275 100644 --- a/arch/x86/lib/msr-reg.S +++ b/arch/x86/lib/msr-reg.S @@ -37,7 +37,7 @@ ENTRY(\op\()_safe_regs) movl %edi, 28(%r10) popq_cfi %rbp popq_cfi %rbx - ret + RET 3: CFI_RESTORE_STATE movl $-EIO, %r11d @@ -84,7 +84,7 @@ ENTRY(\op\()_safe_regs) popl_cfi %esi popl_cfi %ebp popl_cfi %ebx - ret + RET 3: CFI_RESTORE_STATE movl $-EIO, 4(%esp) diff --git a/arch/x86/lib/putuser.S b/arch/x86/lib/putuser.S index fc6ba17a7eec2a..0f097e346f428e 100644 --- a/arch/x86/lib/putuser.S +++ b/arch/x86/lib/putuser.S @@ -33,7 +33,7 @@ #define ENTER CFI_STARTPROC ; \ GET_THREAD_INFO(%_ASM_BX) #define EXIT ASM_CLAC ; \ - ret ; \ + RET ; \ CFI_ENDPROC .text diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index b3bec3619100b2..24a621672402de 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -41,3 +41,66 @@ GENERATE_THUNK(r13) GENERATE_THUNK(r14) GENERATE_THUNK(r15) #endif + +/* + * This function name is magical and is used by -mfunction-return=thunk-extern + * for the compiler to generate JMPs to it. + */ + +/* + * Safety details here pertain to the AMD Zen{1,2} microarchitecture: + * 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for + * alignment within the BTB. + * 2) The instruction at zen_untrain_ret must contain, and not + * end with, the 0xc3 byte of the RET. + * 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread + * from re-poisioning the BTB prediction. + */ + .align 64 + .skip 63, 0xcc +GLOBAL(zen_untrain_ret) + /* + * As executed from zen_untrain_ret, this is: + * + * TEST $0xcc, %bl + * LFENCE + * JMP __x86_return_thunk + * + * Executing the TEST instruction has a side effect of evicting any BTB + * prediction (potentially attacker controlled) attached to the RET, as + * __x86_return_thunk + 1 isn't an instruction boundary at the moment. + */ + .byte 0xf6 + + /* + * As executed from __x86_return_thunk, this is a plain RET. + * + * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8. + * + * We subsequently jump backwards and architecturally execute the RET. + * This creates a correct BTB prediction (type=ret), but in the + * meantime we suffer Straight Line Speculation (because the type was + * no branch) which is halted by the INT3. + * + * With SMT enabled and STIBP active, a sibling thread cannot poison + * RET's prediction to a type of its choice, but can evict the + * prediction due to competitive sharing. If the prediction is + * evicted, __x86_return_thunk will suffer Straight Line Speculation + * which will be contained safely by the INT3. + */ +GLOBAL(__x86_return_thunk) + ret + int3 +ENDPROC(__x86_return_thunk) + /* + * Ensure the TEST decoding / BTB invalidation is complete. + */ + lfence + + /* + * Jump back and execute the RET in the middle of the TEST instruction. + * INT3 is for SLS protection. + */ + jmp __x86_return_thunk + int3 +ENDPROC(zen_untrain_ret) diff --git a/arch/x86/lib/rwlock.S b/arch/x86/lib/rwlock.S index cc4a8c2f9b8d56..13cfc4effe527f 100644 --- a/arch/x86/lib/rwlock.S +++ b/arch/x86/lib/rwlock.S @@ -23,7 +23,7 @@ ENTRY(__write_lock_failed) WRITE_LOCK_SUB($RW_LOCK_BIAS) (%__lock_ptr) jnz 0b FRAME_END - ret + RET CFI_ENDPROC END(__write_lock_failed) @@ -39,6 +39,6 @@ ENTRY(__read_lock_failed) READ_LOCK_SIZE(dec) (%__lock_ptr) js 0b FRAME_END - ret + RET CFI_ENDPROC END(__read_lock_failed) diff --git a/arch/x86/lib/rwsem.S b/arch/x86/lib/rwsem.S index 880fcda587bf47..d541214dba6b21 100644 --- a/arch/x86/lib/rwsem.S +++ b/arch/x86/lib/rwsem.S @@ -97,7 +97,7 @@ ENTRY(call_rwsem_down_read_failed) CFI_RESTORE __ASM_REG(dx) restore_common_regs FRAME_END - ret + RET CFI_ENDPROC ENDPROC(call_rwsem_down_read_failed) @@ -109,7 +109,7 @@ ENTRY(call_rwsem_down_write_failed) call rwsem_down_write_failed restore_common_regs FRAME_END - ret + RET CFI_ENDPROC ENDPROC(call_rwsem_down_write_failed) @@ -124,7 +124,7 @@ ENTRY(call_rwsem_wake) call rwsem_wake restore_common_regs 1: FRAME_END - ret + RET CFI_ENDPROC ENDPROC(call_rwsem_wake) @@ -140,6 +140,6 @@ ENTRY(call_rwsem_downgrade_wake) CFI_RESTORE __ASM_REG(dx) restore_common_regs FRAME_END - ret + RET CFI_ENDPROC ENDPROC(call_rwsem_downgrade_wake) diff --git a/arch/x86/lib/thunk_32.S b/arch/x86/lib/thunk_32.S index 2930ae05d77305..c5fd18f9a29b28 100644 --- a/arch/x86/lib/thunk_32.S +++ b/arch/x86/lib/thunk_32.S @@ -21,7 +21,7 @@ popl %edx popl %ecx popl %eax - ret + RET .endm thunk_ra trace_hardirqs_on_thunk,trace_hardirqs_on_caller diff --git a/arch/x86/lib/thunk_64.S b/arch/x86/lib/thunk_64.S index 227a5328419ff2..db7daa911c44fc 100644 --- a/arch/x86/lib/thunk_64.S +++ b/arch/x86/lib/thunk_64.S @@ -80,6 +80,6 @@ restore: popq_cfi %rdi CFI_RESTORE rdi popq %rbp - ret + RET CFI_ENDPROC #endif diff --git a/arch/x86/math-emu/div_Xsig.S b/arch/x86/math-emu/div_Xsig.S index f77ba3058b3174..ec60fc7faa2c5c 100644 --- a/arch/x86/math-emu/div_Xsig.S +++ b/arch/x86/math-emu/div_Xsig.S @@ -340,7 +340,7 @@ L_exit: popl %esi leave - ret + RET #ifdef PARANOID diff --git a/arch/x86/math-emu/div_small.S b/arch/x86/math-emu/div_small.S index 47099628fa4ceb..f18dca14d6cfb0 100644 --- a/arch/x86/math-emu/div_small.S +++ b/arch/x86/math-emu/div_small.S @@ -43,5 +43,5 @@ ENTRY(FPU_div_small) popl %esi leave - ret + RET diff --git a/arch/x86/math-emu/mul_Xsig.S b/arch/x86/math-emu/mul_Xsig.S index 717785a53eb401..2e57cdaf131581 100644 --- a/arch/x86/math-emu/mul_Xsig.S +++ b/arch/x86/math-emu/mul_Xsig.S @@ -61,7 +61,7 @@ ENTRY(mul32_Xsig) popl %esi leave - ret + RET ENTRY(mul64_Xsig) @@ -113,7 +113,7 @@ ENTRY(mul64_Xsig) popl %esi leave - ret + RET @@ -172,5 +172,5 @@ ENTRY(mul_Xsig_Xsig) popl %esi leave - ret + RET diff --git a/arch/x86/math-emu/polynom_Xsig.S b/arch/x86/math-emu/polynom_Xsig.S index 17315c89ff3d07..82b109b249c56f 100644 --- a/arch/x86/math-emu/polynom_Xsig.S +++ b/arch/x86/math-emu/polynom_Xsig.S @@ -132,4 +132,4 @@ L_accum_done: popl %edi popl %esi leave - ret + RET diff --git a/arch/x86/math-emu/reg_norm.S b/arch/x86/math-emu/reg_norm.S index 8b6352efceef26..76960a452fb447 100644 --- a/arch/x86/math-emu/reg_norm.S +++ b/arch/x86/math-emu/reg_norm.S @@ -71,7 +71,7 @@ L_exit_valid: L_exit: popl %ebx leave - ret + RET L_zero: @@ -136,7 +136,7 @@ L_exit_nuo_valid: popl %ebx leave - ret + RET L_exit_nuo_zero: movl TAG_Zero,%eax @@ -144,4 +144,4 @@ L_exit_nuo_zero: popl %ebx leave - ret + RET diff --git a/arch/x86/math-emu/reg_round.S b/arch/x86/math-emu/reg_round.S index d1d4e48b4f67d2..9aee9e2059dfa8 100644 --- a/arch/x86/math-emu/reg_round.S +++ b/arch/x86/math-emu/reg_round.S @@ -436,7 +436,7 @@ fpu_Arith_exit: popl %edi popl %esi leave - ret + RET /* diff --git a/arch/x86/math-emu/reg_u_add.S b/arch/x86/math-emu/reg_u_add.S index 47c4c2434d85ef..dfc70b0a2917d6 100644 --- a/arch/x86/math-emu/reg_u_add.S +++ b/arch/x86/math-emu/reg_u_add.S @@ -163,5 +163,5 @@ L_exit: popl %edi popl %esi leave - ret + RET #endif /* PARANOID */ diff --git a/arch/x86/math-emu/reg_u_div.S b/arch/x86/math-emu/reg_u_div.S index cc00654b6f9ada..35d808a5488350 100644 --- a/arch/x86/math-emu/reg_u_div.S +++ b/arch/x86/math-emu/reg_u_div.S @@ -467,5 +467,5 @@ L_exit: popl %esi leave - ret + RET #endif /* PARANOID */ diff --git a/arch/x86/math-emu/reg_u_mul.S b/arch/x86/math-emu/reg_u_mul.S index 973f12af97df55..d2f410808bd1b9 100644 --- a/arch/x86/math-emu/reg_u_mul.S +++ b/arch/x86/math-emu/reg_u_mul.S @@ -143,6 +143,6 @@ L_exit: popl %edi popl %esi leave - ret + RET #endif /* PARANOID */ diff --git a/arch/x86/math-emu/reg_u_sub.S b/arch/x86/math-emu/reg_u_sub.S index 1b6c24801d2231..9abc5274028afa 100644 --- a/arch/x86/math-emu/reg_u_sub.S +++ b/arch/x86/math-emu/reg_u_sub.S @@ -269,4 +269,4 @@ L_exit: popl %edi popl %esi leave - ret + RET diff --git a/arch/x86/math-emu/round_Xsig.S b/arch/x86/math-emu/round_Xsig.S index bbe0e87718e428..857a94475f086c 100644 --- a/arch/x86/math-emu/round_Xsig.S +++ b/arch/x86/math-emu/round_Xsig.S @@ -77,7 +77,7 @@ L_exit: popl %esi popl %ebx leave - ret + RET @@ -137,5 +137,5 @@ L_n_exit: popl %esi popl %ebx leave - ret + RET diff --git a/arch/x86/math-emu/shr_Xsig.S b/arch/x86/math-emu/shr_Xsig.S index 31cdd118e9186c..948924eca3dbfc 100644 --- a/arch/x86/math-emu/shr_Xsig.S +++ b/arch/x86/math-emu/shr_Xsig.S @@ -44,7 +44,7 @@ ENTRY(shr_Xsig) popl %ebx popl %esi leave - ret + RET L_more_than_31: cmpl $64,%ecx @@ -60,7 +60,7 @@ L_more_than_31: movl $0,8(%esi) popl %esi leave - ret + RET L_more_than_63: cmpl $96,%ecx @@ -75,7 +75,7 @@ L_more_than_63: movl %edx,8(%esi) popl %esi leave - ret + RET L_more_than_95: xorl %eax,%eax @@ -84,4 +84,4 @@ L_more_than_95: movl %eax,8(%esi) popl %esi leave - ret + RET diff --git a/arch/x86/math-emu/wm_shrx.S b/arch/x86/math-emu/wm_shrx.S index 51842831798599..bb88cf6203fadd 100644 --- a/arch/x86/math-emu/wm_shrx.S +++ b/arch/x86/math-emu/wm_shrx.S @@ -54,7 +54,7 @@ ENTRY(FPU_shrx) popl %ebx popl %esi leave - ret + RET L_more_than_31: cmpl $64,%ecx @@ -69,7 +69,7 @@ L_more_than_31: movl $0,4(%esi) popl %esi leave - ret + RET L_more_than_63: cmpl $96,%ecx @@ -83,7 +83,7 @@ L_more_than_63: movl %edx,4(%esi) popl %esi leave - ret + RET L_more_than_95: xorl %eax,%eax @@ -91,7 +91,7 @@ L_more_than_95: movl %eax,4(%esi) popl %esi leave - ret + RET /*---------------------------------------------------------------------------+ @@ -144,7 +144,7 @@ ENTRY(FPU_shrxs) popl %ebx popl %esi leave - ret + RET /* Shift by [0..31] bits */ Ls_less_than_32: @@ -161,7 +161,7 @@ Ls_less_than_32: popl %ebx popl %esi leave - ret + RET /* Shift by [64..95] bits */ Ls_more_than_63: @@ -187,7 +187,7 @@ Ls_more_than_63: popl %ebx popl %esi leave - ret + RET Ls_more_than_95: /* Shift by [96..inf) bits */ @@ -201,4 +201,4 @@ Ls_more_than_95: popl %ebx popl %esi leave - ret + RET diff --git a/arch/x86/mm/mem_encrypt_boot.S b/arch/x86/mm/mem_encrypt_boot.S index 01f682cf77a8b3..d63da82337ee5e 100644 --- a/arch/x86/mm/mem_encrypt_boot.S +++ b/arch/x86/mm/mem_encrypt_boot.S @@ -66,7 +66,7 @@ ENTRY(sme_encrypt_execute) movq %rbp, %rsp /* Restore original stack pointer */ pop %rbp - ret + RET ENDPROC(sme_encrypt_execute) ENTRY(__enc_copy) @@ -152,6 +152,6 @@ ENTRY(__enc_copy) pop %r12 pop %r15 - ret + RET .L__enc_copy_end: ENDPROC(__enc_copy) diff --git a/arch/x86/net/bpf_jit.S b/arch/x86/net/bpf_jit.S index 68b0ebbcfdb0cb..ae7772a7fa8f9b 100644 --- a/arch/x86/net/bpf_jit.S +++ b/arch/x86/net/bpf_jit.S @@ -37,7 +37,7 @@ FUNC(sk_load_word_positive_offset) jle bpf_slow_path_word mov (SKBDATA,%rsi),%eax bswap %eax /* ntohl() */ - ret + RET FUNC(sk_load_half) test %esi,%esi @@ -50,7 +50,7 @@ FUNC(sk_load_half_positive_offset) jle bpf_slow_path_half movzwl (SKBDATA,%rsi),%eax rol $8,%ax # ntohs() - ret + RET FUNC(sk_load_byte) test %esi,%esi @@ -60,7 +60,7 @@ FUNC(sk_load_byte_positive_offset) cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */ jle bpf_slow_path_byte movzbl (SKBDATA,%rsi),%eax - ret + RET /** * sk_load_byte_msh - BPF_S_LDX_B_MSH helper @@ -80,7 +80,7 @@ FUNC(sk_load_byte_msh_positive_offset) movzbl (SKBDATA,%rsi),%ebx and $15,%bl shl $2,%bl - ret + RET /* rsi contains offset and can be scratched */ #define bpf_slow_path_common(LEN) \ @@ -104,7 +104,7 @@ bpf_slow_path_word: js bpf_error mov -12(%rbp),%eax bswap %eax - ret + RET bpf_slow_path_half: bpf_slow_path_common(2) @@ -112,13 +112,13 @@ bpf_slow_path_half: mov -12(%rbp),%ax rol $8,%ax movzwl %ax,%eax - ret + RET bpf_slow_path_byte: bpf_slow_path_common(1) js bpf_error movzbl -12(%rbp),%eax - ret + RET bpf_slow_path_byte_msh: xchg %eax,%ebx /* dont lose A , X is about to be scratched */ @@ -128,7 +128,7 @@ bpf_slow_path_byte_msh: and $15,%al shl $2,%al xchg %eax,%ebx - ret + RET #define sk_negative_common(SIZE) \ FRAME_BEGIN; \ @@ -154,7 +154,7 @@ FUNC(sk_load_word_negative_offset) sk_negative_common(4) mov (%rax), %eax bswap %eax - ret + RET bpf_slow_path_half_neg: cmp SKF_MAX_NEG_OFF, %esi @@ -165,7 +165,7 @@ FUNC(sk_load_half_negative_offset) mov (%rax),%ax rol $8,%ax movzwl %ax,%eax - ret + RET bpf_slow_path_byte_neg: cmp SKF_MAX_NEG_OFF, %esi @@ -174,7 +174,7 @@ bpf_slow_path_byte_neg: FUNC(sk_load_byte_negative_offset) sk_negative_common(1) movzbl (%rax), %eax - ret + RET bpf_slow_path_byte_msh_neg: cmp SKF_MAX_NEG_OFF, %esi @@ -187,11 +187,11 @@ FUNC(sk_load_byte_msh_negative_offset) and $15,%al shl $2,%al xchg %eax,%ebx - ret + RET bpf_error: # force a return 0 from jit handler xor %eax,%eax mov -8(%rbp),%rbx leaveq - ret + RET diff --git a/arch/x86/net/trace_bpf_jit.S b/arch/x86/net/trace_bpf_jit.S index 9ffd594e9f6229..c52da0c41eae9c 100644 --- a/arch/x86/net/trace_bpf_jit.S +++ b/arch/x86/net/trace_bpf_jit.S @@ -36,7 +36,7 @@ FUNC(trace_sk_load_word_positive_offset) jle bpf_slow_path_word mov (SKBDATA,%rsi),%eax bswap %eax /* ntohl() */ - ret + RET FUNC(trace_sk_load_half) test %esi,%esi @@ -49,7 +49,7 @@ FUNC(trace_sk_load_half_positive_offset) jle bpf_slow_path_half movzwl (SKBDATA,%rsi),%eax rol $8,%ax # ntohs() - ret + RET FUNC(trace_sk_load_byte) test %esi,%esi @@ -59,7 +59,7 @@ FUNC(trace_sk_load_byte_positive_offset) cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */ jle bpf_slow_path_byte movzbl (SKBDATA,%rsi),%eax - ret + RET /* rsi contains offset and can be scratched */ #define bpf_slow_path_common(LEN) \ @@ -82,7 +82,7 @@ bpf_slow_path_word: js bpf_error mov 32(%rbp),%eax bswap %eax - ret + RET bpf_slow_path_half: bpf_slow_path_common(2) @@ -90,13 +90,13 @@ bpf_slow_path_half: mov 32(%rbp),%ax rol $8,%ax movzwl %ax,%eax - ret + RET bpf_slow_path_byte: bpf_slow_path_common(1) js bpf_error movzbl 32(%rbp),%eax - ret + RET #define sk_negative_common(SIZE) \ FRAME_BEGIN; \ @@ -120,7 +120,7 @@ FUNC(trace_sk_load_word_negative_offset) sk_negative_common(4) mov (%rax), %eax bswap %eax - ret + RET bpf_slow_path_half_neg: cmp SKF_MAX_NEG_OFF, %esi @@ -131,7 +131,7 @@ FUNC(trace_sk_load_half_negative_offset) mov (%rax),%ax rol $8,%ax movzwl %ax,%eax - ret + RET bpf_slow_path_byte_neg: cmp SKF_MAX_NEG_OFF, %esi @@ -140,7 +140,7 @@ bpf_slow_path_byte_neg: FUNC(trace_sk_load_byte_negative_offset) sk_negative_common(1) movzbl (%rax), %eax - ret + RET bpf_error: # force a return 0 from jit handler @@ -151,4 +151,4 @@ bpf_error: mov 24(%rbp),%r15 add $40, %rbp leaveq - ret + RET diff --git a/arch/x86/platform/olpc/xo1-wakeup.S b/arch/x86/platform/olpc/xo1-wakeup.S index 948deb289753f8..ec0132c5207bdc 100644 --- a/arch/x86/platform/olpc/xo1-wakeup.S +++ b/arch/x86/platform/olpc/xo1-wakeup.S @@ -76,7 +76,7 @@ save_registers: pushfl popl saved_context_eflags - ret + RET restore_registers: movl saved_context_ebp, %ebp @@ -87,7 +87,7 @@ restore_registers: pushl saved_context_eflags popfl - ret + RET ENTRY(do_olpc_suspend_lowlevel) call save_processor_state @@ -108,7 +108,7 @@ ret_point: call restore_registers call restore_processor_state - ret + RET .data saved_gdt: .long 0,0 diff --git a/arch/x86/power/hibernate_asm_32.S b/arch/x86/power/hibernate_asm_32.S index 1d0fa0e24070bb..8a0c89ff46debd 100644 --- a/arch/x86/power/hibernate_asm_32.S +++ b/arch/x86/power/hibernate_asm_32.S @@ -24,7 +24,7 @@ ENTRY(swsusp_arch_suspend) popl saved_context_eflags call swsusp_save - ret + RET ENTRY(restore_image) movl mmu_cr4_features, %ecx @@ -81,4 +81,4 @@ done: xorl %eax, %eax - ret + RET diff --git a/arch/x86/power/hibernate_asm_64.S b/arch/x86/power/hibernate_asm_64.S index ce8da3a0412cbb..80486ed6a026ac 100644 --- a/arch/x86/power/hibernate_asm_64.S +++ b/arch/x86/power/hibernate_asm_64.S @@ -50,7 +50,7 @@ ENTRY(swsusp_arch_suspend) FRAME_BEGIN call swsusp_save FRAME_END - ret + RET ENDPROC(swsusp_arch_suspend) ENTRY(restore_image) @@ -142,5 +142,5 @@ ENTRY(restore_registers) /* tell the hibernation core that we've just restored the memory */ movq %rax, in_suspend(%rip) - ret + RET ENDPROC(restore_registers) diff --git a/arch/x86/um/checksum_32.S b/arch/x86/um/checksum_32.S index 8d0c420465cce0..e8da46c61222a2 100644 --- a/arch/x86/um/checksum_32.S +++ b/arch/x86/um/checksum_32.S @@ -113,7 +113,7 @@ csum_partial: 7: popl %ebx popl %esi - ret + RET #else @@ -211,7 +211,7 @@ csum_partial: 80: popl %ebx popl %esi - ret + RET #endif @@ -363,7 +363,7 @@ DST( movb %cl, (%edi) ) popl %esi popl %edi popl %ecx # equivalent to addl $4,%esp - ret + RET #else @@ -447,7 +447,7 @@ DST( movb %dl, (%edi) ) popl %esi popl %edi popl %ebx - ret + RET #undef ROUND #undef ROUND1 diff --git a/arch/x86/um/setjmp_32.S b/arch/x86/um/setjmp_32.S index b766792c99335a..a27b1da7252221 100644 --- a/arch/x86/um/setjmp_32.S +++ b/arch/x86/um/setjmp_32.S @@ -33,7 +33,7 @@ setjmp: movl %esi,12(%edx) movl %edi,16(%edx) movl %ecx,20(%edx) # Return address - ret + RET .size setjmp,.-setjmp diff --git a/arch/x86/um/setjmp_64.S b/arch/x86/um/setjmp_64.S index 45f547b4043eed..e97fae42a0d15b 100644 --- a/arch/x86/um/setjmp_64.S +++ b/arch/x86/um/setjmp_64.S @@ -32,7 +32,7 @@ setjmp: movq %r14,40(%rdi) movq %r15,48(%rdi) movq %rsi,56(%rdi) # Return address - ret + RET .size setjmp,.-setjmp diff --git a/arch/x86/vdso/Makefile b/arch/x86/vdso/Makefile index 10596e431f0e41..5e0e8ec03878da 100644 --- a/arch/x86/vdso/Makefile +++ b/arch/x86/vdso/Makefile @@ -62,7 +62,14 @@ CFL := $(PROFILING) -mcmodel=small -fPIC -O2 -fasynchronous-unwind-tables -m64 \ $(filter -g%,$(KBUILD_CFLAGS)) $(call cc-option, -fno-stack-protector) \ -fno-omit-frame-pointer -foptimize-sibling-calls -$(vobjs): KBUILD_CFLAGS += $(CFL) +ifdef CONFIG_RETPOLINE +ifneq ($(RETPOLINE_VDSO_CFLAGS),) + CFL += $(RETPOLINE_VDSO_CFLAGS) +endif +endif + +$(vobjs): KBUILD_CFLAGS := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL) +$(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO # # vDSO code runs in userspace and -pg doesn't help with profiling anyway. @@ -150,6 +157,7 @@ extra-y += $(vdso32-images) $(obj)/vdso32.o: $(vdso32-images:%=$(obj)/%) KBUILD_AFLAGS_32 := $(filter-out -m64,$(KBUILD_AFLAGS)) +KBUILD_AFLAGS_32 += -DBUILD_VDSO $(vdso32-images:%=$(obj)/%.dbg): KBUILD_AFLAGS = $(KBUILD_AFLAGS_32) $(vdso32-images:%=$(obj)/%.dbg): asflags-$(CONFIG_X86_64) += -m32 diff --git a/arch/x86/vdso/vdso32/int80.S b/arch/x86/vdso/vdso32/int80.S index b15b7c01aedbea..9606c3cb24abf9 100644 --- a/arch/x86/vdso/vdso32/int80.S +++ b/arch/x86/vdso/vdso32/int80.S @@ -13,7 +13,7 @@ __kernel_vsyscall: .LSTART_vsyscall: int $0x80 - ret + RET .LEND_vsyscall: .size __kernel_vsyscall,.-.LSTART_vsyscall .previous diff --git a/arch/x86/vdso/vdso32/syscall.S b/arch/x86/vdso/vdso32/syscall.S index 5415b5613d5545..9ffc2751058953 100644 --- a/arch/x86/vdso/vdso32/syscall.S +++ b/arch/x86/vdso/vdso32/syscall.S @@ -24,7 +24,7 @@ __kernel_vsyscall: movl %ebp, %ecx popl %ebp .Lpop_ebp: - ret + RET .LEND_vsyscall: .size __kernel_vsyscall,.-.LSTART_vsyscall diff --git a/arch/x86/vdso/vdso32/sysenter.S b/arch/x86/vdso/vdso32/sysenter.S index e354bceee0e049..8f26aa37585c11 100644 --- a/arch/x86/vdso/vdso32/sysenter.S +++ b/arch/x86/vdso/vdso32/sysenter.S @@ -52,7 +52,7 @@ VDSO32_SYSENTER_RETURN: /* Symbol used by sysenter.c via vdso32-syms.h */ .Lpop_edx: pop %ecx .Lpop_ecx: - ret + RET .LEND_vsyscall: .size __kernel_vsyscall,.-.LSTART_vsyscall .previous diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S index eff224df813fdd..4214517fee1c47 100644 --- a/arch/x86/xen/xen-asm.S +++ b/arch/x86/xen/xen-asm.S @@ -42,7 +42,7 @@ ENTRY(xen_irq_enable_direct) 1: ENDPATCH(xen_irq_enable_direct) FRAME_END - ret + RET ENDPROC(xen_irq_enable_direct) RELOC(xen_irq_enable_direct, 2b+1) @@ -54,7 +54,7 @@ ENDPATCH(xen_irq_enable_direct) ENTRY(xen_irq_disable_direct) movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask ENDPATCH(xen_irq_disable_direct) - ret + RET ENDPROC(xen_irq_disable_direct) RELOC(xen_irq_disable_direct, 0) @@ -72,7 +72,7 @@ ENTRY(xen_save_fl_direct) setz %ah addb %ah, %ah ENDPATCH(xen_save_fl_direct) - ret + RET ENDPROC(xen_save_fl_direct) RELOC(xen_save_fl_direct, 0) @@ -105,7 +105,7 @@ ENTRY(xen_restore_fl_direct) 1: ENDPATCH(xen_restore_fl_direct) FRAME_END - ret + RET ENDPROC(xen_restore_fl_direct) RELOC(xen_restore_fl_direct, 2b+1) @@ -146,5 +146,5 @@ ENTRY(check_events) pop %rax #endif FRAME_END - ret + RET ENDPROC(check_events) diff --git a/arch/x86/xen/xen-asm_32.S b/arch/x86/xen/xen-asm_32.S index 33ca6e42a4caab..ae9a7a0fafa339 100644 --- a/arch/x86/xen/xen-asm_32.S +++ b/arch/x86/xen/xen-asm_32.S @@ -32,7 +32,7 @@ check_events: pop %edx pop %ecx pop %eax - ret + RET /* * We can't use sysexit directly, because we're not running in ring0. diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index 1624bfc767a923..96f519275438ed 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -457,6 +457,12 @@ ssize_t __weak cpu_show_mmio_stale_data(struct device *dev, return sprintf(buf, "Not affected\n"); } +ssize_t __weak cpu_show_retbleed(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "Not affected\n"); +} + static DEVICE_ATTR(meltdown, 0400, cpu_show_meltdown, NULL); static DEVICE_ATTR(spectre_v1, 0400, cpu_show_spectre_v1, NULL); static DEVICE_ATTR(spectre_v2, 0400, cpu_show_spectre_v2, NULL); @@ -467,6 +473,7 @@ static DEVICE_ATTR(tsx_async_abort, 0400, cpu_show_tsx_async_abort, NULL); static DEVICE_ATTR(itlb_multihit, 0400, cpu_show_itlb_multihit, NULL); static DEVICE_ATTR(srbds, 0400, cpu_show_srbds, NULL); static DEVICE_ATTR(mmio_stale_data, 0400, cpu_show_mmio_stale_data, NULL); +static DEVICE_ATTR(retbleed, 0400, cpu_show_retbleed, NULL); static struct attribute *cpu_root_vulnerabilities_attrs[] = { &dev_attr_meltdown.attr, @@ -479,6 +486,7 @@ static struct attribute *cpu_root_vulnerabilities_attrs[] = { &dev_attr_itlb_multihit.attr, &dev_attr_srbds.attr, &dev_attr_mmio_stale_data.attr, + &dev_attr_retbleed.attr, NULL }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index aae185bfc3d3ea..83d92ec18b06c5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -844,6 +844,7 @@ static void cmd_work_handler(struct work_struct *work) struct mlx5_cmd_layout *lay; struct semaphore *sem; unsigned long flags; + bool poll_cmd = ent->polling; int alloc_ret; int cmd_mode; @@ -917,7 +918,7 @@ static void cmd_work_handler(struct work_struct *work) iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell); mmiowb(); /* if not in polling don't use ent after this point */ - if (cmd_mode == CMD_MODE_POLLING) { + if (cmd_mode == CMD_MODE_POLLING || poll_cmd) { poll_timeout(ent); /* make sure we read the descriptor after ownership is SW */ rmb(); @@ -961,7 +962,7 @@ static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) struct mlx5_cmd *cmd = &dev->cmd; int err; - if (cmd->mode == CMD_MODE_POLLING) { + if (cmd->mode == CMD_MODE_POLLING || ent->polling) { wait_for_completion(&ent->done); } else if (!wait_for_completion_timeout(&ent->done, timeout)) { ent->ret = -ETIMEDOUT; @@ -989,7 +990,7 @@ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, struct mlx5_cmd_msg *out, void *uout, int uout_size, mlx5_cmd_cbk_t callback, void *context, int page_queue, u8 *status, - u8 token) + u8 token, bool force_polling) { struct mlx5_cmd *cmd = &dev->cmd; struct mlx5_cmd_work_ent *ent; @@ -1007,6 +1008,7 @@ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, return PTR_ERR(ent); ent->token = token; + ent->polling = force_polling; if (!callback) init_completion(&ent->done); @@ -1591,7 +1593,8 @@ static int is_manage_pages(void *in) } static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, - int out_size, mlx5_cmd_cbk_t callback, void *context) + int out_size, mlx5_cmd_cbk_t callback, void *context, + bool force_polling) { struct mlx5_cmd_msg *inb; struct mlx5_cmd_msg *outb; @@ -1636,7 +1639,7 @@ static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, } err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context, - pages_queue, &status, token); + pages_queue, &status, token, force_polling); if (err) goto out_out; @@ -1664,7 +1667,7 @@ int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, { int err; - err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL); + err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false); return err ? : mlx5_cmd_check(dev, in, out); } EXPORT_SYMBOL(mlx5_cmd_exec); @@ -1673,10 +1676,22 @@ int mlx5_cmd_exec_cb(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size, mlx5_cmd_cbk_t callback, void *context) { - return cmd_exec(dev, in, in_size, out, out_size, callback, context); + return cmd_exec(dev, in, in_size, out, out_size, callback, context, + false); } EXPORT_SYMBOL(mlx5_cmd_exec_cb); +int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size, + void *out, int out_size) +{ + int err; + + err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true); + + return err ? : mlx5_cmd_check(dev, in, out); +} +EXPORT_SYMBOL(mlx5_cmd_exec_polling); + static void destroy_msg_cache(struct mlx5_core_dev *dev) { struct cmd_msg_cache *ch; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index fa179302120944..a2a14b472245be 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -1094,6 +1094,33 @@ struct mlx5_eq_comp *mlx5_eqn2comp_eq(struct mlx5_core_dev *dev, int eqn) return ERR_PTR(-ENOENT); } +/* This function should only be called after mlx5_cmd_force_teardown_hca */ +void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev) +{ + struct mlx5_eq_table *table = dev->priv.eq_table; + int i, max_eqs; + + clear_comp_irqs_affinity_hints(dev); + +#ifdef CONFIG_RFS_ACCEL + if (table->rmap) { + free_irq_cpu_rmap(table->rmap); + table->rmap = NULL; + } +#endif + + mutex_lock(&table->lock); /* sync with create/destroy_async_eq */ + max_eqs = table->num_comp_vectors + MLX5_EQ_VEC_COMP_BASE; + for (i = max_eqs - 1; i >= 0; i--) { + if (!table->irq_info[i].context) + continue; + free_irq(pci_irq_vector(dev->pdev, i), table->irq_info[i].context); + table->irq_info[i].context = NULL; + } + mutex_unlock(&table->lock); + pci_free_irq_vectors(dev->pdev); +} + static int alloc_irq_vectors(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index d537e62e868a07..1ab6f7e3bec626 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -231,6 +231,82 @@ int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev) return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } +int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev) +{ + u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {0}; + u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {0}; + int force_state; + int ret; + + if (!MLX5_CAP_GEN(dev, force_teardown)) { + mlx5_core_dbg(dev, "force teardown is not supported in the firmware\n"); + return -EOPNOTSUPP; + } + + MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA); + MLX5_SET(teardown_hca_in, in, profile, MLX5_TEARDOWN_HCA_IN_PROFILE_FORCE_CLOSE); + + ret = mlx5_cmd_exec_polling(dev, in, sizeof(in), out, sizeof(out)); + if (ret) + return ret; + + force_state = MLX5_GET(teardown_hca_out, out, state); + if (force_state == MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL) { + mlx5_core_warn(dev, "teardown with force mode failed, doing normal teardown\n"); + return -EIO; + } + + return 0; +} + +#define MLX5_FAST_TEARDOWN_WAIT_MS 3000 +int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev) +{ + unsigned long end, delay_ms = MLX5_FAST_TEARDOWN_WAIT_MS; + u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {0}; + u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {0}; + int state; + int ret; + + if (!MLX5_CAP_GEN(dev, fast_teardown)) { + mlx5_core_dbg(dev, "fast teardown is not supported in the firmware\n"); + return -EOPNOTSUPP; + } + + MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA); + MLX5_SET(teardown_hca_in, in, profile, + MLX5_TEARDOWN_HCA_IN_PROFILE_PREPARE_FAST_TEARDOWN); + + ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (ret) + return ret; + + state = MLX5_GET(teardown_hca_out, out, state); + if (state == MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL) { + mlx5_core_warn(dev, "teardown with fast mode failed\n"); + return -EIO; + } + + mlx5_set_nic_state(dev, MLX5_NIC_IFC_DISABLED); + + /* Loop until device state turns to disable */ + end = jiffies + msecs_to_jiffies(delay_ms); + do { + if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) + break; + + cond_resched(); + } while (!time_after(jiffies, end)); + + if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) { + dev_err(&dev->pdev->dev, "NIC IFC still %d after %lums.\n", + mlx5_get_nic_state(dev), delay_ms); + return -EIO; + } + + return 0; +} + enum mlxsw_reg_mcc_instruction { MLX5_REG_MCC_INSTRUCTION_LOCK_UPDATE_HANDLE = 0x01, MLX5_REG_MCC_INSTRUCTION_RELEASE_UPDATE_HANDLE = 0x02, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index 7004ce46f35427..066883003aeaa0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -59,23 +59,26 @@ enum { MLX5_HEALTH_SYNDR_HIGH_TEMP = 0x10 }; -enum { - MLX5_NIC_IFC_FULL = 0, - MLX5_NIC_IFC_DISABLED = 1, - MLX5_NIC_IFC_NO_DRAM_NIC = 2, - MLX5_NIC_IFC_INVALID = 3 -}; - enum { MLX5_DROP_NEW_HEALTH_WORK, MLX5_DROP_NEW_RECOVERY_WORK, }; -static u8 get_nic_state(struct mlx5_core_dev *dev) +u8 mlx5_get_nic_state(struct mlx5_core_dev *dev) { return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3; } +void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state) +{ + u32 cur_cmdq_addr_l_sz; + + cur_cmdq_addr_l_sz = ioread32be(&dev->iseg->cmdq_addr_l_sz); + iowrite32be((cur_cmdq_addr_l_sz & 0xFFFFF000) | + state << MLX5_NIC_IFC_OFFSET, + &dev->iseg->cmdq_addr_l_sz); +} + static void trigger_cmd_completions(struct mlx5_core_dev *dev) { unsigned long flags; @@ -104,7 +107,7 @@ static int in_fatal(struct mlx5_core_dev *dev) struct mlx5_core_health *health = &dev->priv.health; struct health_buffer __iomem *h = health->health; - if (get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) + if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) return 1; if (ioread32be(&h->fw_ver) == 0xffffffff) @@ -113,14 +116,14 @@ static int in_fatal(struct mlx5_core_dev *dev) return 0; } -void mlx5_enter_error_state(struct mlx5_core_dev *dev) +void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) { mutex_lock(&dev->intf_state_mutex); if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) goto unlock; mlx5_core_err(dev, "start\n"); - if (pci_channel_offline(dev->pdev) || in_fatal(dev)) { + if (pci_channel_offline(dev->pdev) || in_fatal(dev) || force) { dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; trigger_cmd_completions(dev); } @@ -134,7 +137,7 @@ void mlx5_enter_error_state(struct mlx5_core_dev *dev) static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) { - u8 nic_interface = get_nic_state(dev); + u8 nic_interface = mlx5_get_nic_state(dev); switch (nic_interface) { case MLX5_NIC_IFC_FULL: @@ -169,7 +172,7 @@ static void health_recover(struct work_struct *work) priv = container_of(health, struct mlx5_priv, health); dev = container_of(priv, struct mlx5_core_dev, priv); - nic_state = get_nic_state(dev); + nic_state = mlx5_get_nic_state(dev); if (nic_state == MLX5_NIC_IFC_INVALID) { dev_err(&dev->pdev->dev, "health recovery flow aborted since the nic state is invalid\n"); return; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index b83bea6e48e53e..5a4c9c3d21590a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1238,7 +1238,7 @@ static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, dev_info(&pdev->dev, "%s was called\n", __func__); - mlx5_enter_error_state(dev); + mlx5_enter_error_state(dev, false); mlx5_unload_one(dev, priv, false); /* In case of kernel call drain the health wq */ if (state) { @@ -1325,13 +1325,66 @@ static const struct pci_error_handlers mlx5_err_handler = { .resume = mlx5_pci_resume }; +static int mlx5_try_fast_unload(struct mlx5_core_dev *dev) +{ + bool fast_teardown = false, force_teardown = false; + int ret = 1; + + fast_teardown = MLX5_CAP_GEN(dev, fast_teardown); + force_teardown = MLX5_CAP_GEN(dev, force_teardown); + + mlx5_core_dbg(dev, "force teardown firmware support=%d\n", force_teardown); + mlx5_core_dbg(dev, "fast teardown firmware support=%d\n", fast_teardown); + + if (!fast_teardown && !force_teardown) + return -EOPNOTSUPP; + + if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { + mlx5_core_dbg(dev, "Device in internal error state, giving up\n"); + return -EAGAIN; + } + + /* Panic tear down fw command will stop the PCI bus communication + * with the HCA, so the health polll is no longer needed. + */ + mlx5_drain_health_wq(dev); + mlx5_stop_health_poll(dev, false); + + ret = mlx5_cmd_fast_teardown_hca(dev); + if (!ret) + goto succeed; + + ret = mlx5_cmd_force_teardown_hca(dev); + if (!ret) + goto succeed; + + mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", ret); + mlx5_start_health_poll(dev); + return ret; + +succeed: + mlx5_enter_error_state(dev, true); + + /* Some platforms requiring freeing the IRQ's in the shutdown + * flow. If they aren't freed they can't be allocated after + * kexec. There is no need to cleanup the mlx5_core software + * contexts. + */ + mlx5_core_eq_free_irqs(dev); + + return 0; +} + static void shutdown(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); struct mlx5_priv *priv = &dev->priv; + int err; dev_info(&pdev->dev, "Shutdown was called\n"); - mlx5_unload_one(dev, priv, false); + err = mlx5_try_fast_unload(dev); + if (err) + mlx5_unload_one(dev, priv, false); mlx5_pci_disable_device(dev); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 72c4903ec42bc6..3405eb3a9efdb0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -99,10 +99,13 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev); int mlx5_query_board_id(struct mlx5_core_dev *dev); int mlx5_cmd_init_hca(struct mlx5_core_dev *dev, uint32_t *sw_owner_id); int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev); +int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev); +int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev); + void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, unsigned long param); void mlx5_port_module_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe); -void mlx5_enter_error_state(struct mlx5_core_dev *dev); +void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force); void mlx5_disable_device(struct mlx5_core_dev *dev); void mlx5_recover_device(struct mlx5_core_dev *dev); int mlx5_sriov_init(struct mlx5_core_dev *dev); @@ -128,6 +131,9 @@ void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced); int mlx5_cq_debugfs_init(struct mlx5_core_dev *dev); void mlx5_cq_debugfs_cleanup(struct mlx5_core_dev *dev); +/* This function should only be called after mlx5_cmd_force_teardown_hca */ +void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev); + int mlx5_query_pcam_reg(struct mlx5_core_dev *dev, u32 *pcam, u8 feature_group, u8 access_reg_group); int mlx5_query_mcam_reg(struct mlx5_core_dev *dev, u32 *mcap, u8 feature_group, @@ -183,4 +189,14 @@ static inline int mlx5_lag_is_lacp_owner(struct mlx5_core_dev *dev) void mlx5_reload_interface(struct mlx5_core_dev *mdev, int protocol); void mlx5_lag_update(struct mlx5_core_dev *dev); + +enum { + MLX5_NIC_IFC_FULL = 0, + MLX5_NIC_IFC_DISABLED = 1, + MLX5_NIC_IFC_NO_DRAM_NIC = 2, + MLX5_NIC_IFC_INVALID = 3 +}; + +u8 mlx5_get_nic_state(struct mlx5_core_dev *dev); +void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state); #endif /* __MLX5_CORE_H__ */ diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index bb1cf126c14f19..d94b88d3684027 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -11362,15 +11362,54 @@ lpfc_sli_hba_iocb_abort(struct lpfc_hba *phba) } /** - * lpfc_sli_validate_fcp_iocb - find commands associated with a vport or LUN + * lpfc_sli_validate_fcp_iocb_for_abort - filter iocbs appropriate for FCP aborts + * @iocbq: Pointer to iocb object. + * @vport: Pointer to driver virtual port object. + * + * This function acts as an iocb filter for functions which abort FCP iocbs. + * + * Return values + * -ENODEV, if a null iocb or vport ptr is encountered + * -EINVAL, if the iocb is not an FCP I/O, not on the TX cmpl queue, premarked as + * driver already started the abort process, or is an abort iocb itself + * 0, passes criteria for aborting the FCP I/O iocb + **/ +static int +lpfc_sli_validate_fcp_iocb_for_abort(struct lpfc_iocbq *iocbq, + struct lpfc_vport *vport) +{ + IOCB_t *icmd = NULL; + + /* No null ptr vports */ + if (!iocbq || iocbq->vport != vport) + return -ENODEV; + + /* iocb must be for FCP IO, already exists on the TX cmpl queue, + * can't be premarked as driver aborted, nor be an ABORT iocb itself + */ + icmd = &iocbq->iocb; + if (!(iocbq->iocb_flag & LPFC_IO_FCP) || + !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ) || + (iocbq->iocb_flag & LPFC_DRIVER_ABORTED) || + (icmd->ulpCommand == CMD_ABORT_XRI_CN || + icmd->ulpCommand == CMD_CLOSE_XRI_CN)) + return -EINVAL; + + return 0; +} + +/** + * lpfc_sli_validate_fcp_iocb - validate commands associated with a SCSI target * @iocbq: Pointer to driver iocb object. * @vport: Pointer to driver virtual port object. * @tgt_id: SCSI ID of the target. * @lun_id: LUN ID of the scsi device. * @ctx_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST * - * This function acts as an iocb filter for functions which abort or count - * all FCP iocbs pending on a lun/SCSI target/SCSI host. It will return + * This function acts as an iocb filter for validating a lun/SCSI target/SCSI + * host. + * + * It will return * 0 if the filtering criteria is met for the given iocb and will return * 1 if the filtering criteria is not met. * If ctx_cmd == LPFC_CTX_LUN, the function returns 0 only if the @@ -11391,12 +11430,6 @@ lpfc_sli_validate_fcp_iocb(struct lpfc_iocbq *iocbq, struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd; int rc = 1; - if (iocbq->vport != vport) - return rc; - - if (!(iocbq->iocb_flag & LPFC_IO_FCP) || - !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ)) - return rc; lpfc_cmd = container_of(iocbq, struct lpfc_scsi_buf, cur_iocbq); @@ -11452,17 +11485,33 @@ lpfc_sli_sum_iocb(struct lpfc_vport *vport, uint16_t tgt_id, uint64_t lun_id, { struct lpfc_hba *phba = vport->phba; struct lpfc_iocbq *iocbq; + IOCB_t *icmd = NULL; int sum, i; + unsigned long iflags; - spin_lock_irq(&phba->hbalock); + spin_lock_irqsave(&phba->hbalock, iflags); for (i = 1, sum = 0; i <= phba->sli.last_iotag; i++) { iocbq = phba->sli.iocbq_lookup[i]; - if (lpfc_sli_validate_fcp_iocb (iocbq, vport, tgt_id, lun_id, - ctx_cmd) == 0) + if (!iocbq || iocbq->vport != vport) + continue; + if (!(iocbq->iocb_flag & LPFC_IO_FCP) || + !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ)) + continue; + + /* Include counting outstanding aborts */ + icmd = &iocbq->iocb; + if (icmd->ulpCommand == CMD_ABORT_XRI_CN || + icmd->ulpCommand == CMD_CLOSE_XRI_CN) { + sum++; + continue; + } + + if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id, + ctx_cmd) == 0) sum++; } - spin_unlock_irq(&phba->hbalock); + spin_unlock_irqrestore(&phba->hbalock, iflags); return sum; } @@ -11503,7 +11552,11 @@ lpfc_sli_abort_fcp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, * * This function sends an abort command for every SCSI command * associated with the given virtual port pending on the ring - * filtered by lpfc_sli_validate_fcp_iocb function. + * filtered by lpfc_sli_validate_fcp_iocb_for_abort and then + * lpfc_sli_validate_fcp_iocb function. The ordering for validation before + * submitting abort iocbs must be lpfc_sli_validate_fcp_iocb_for_abort + * followed by lpfc_sli_validate_fcp_iocb. + * * When abort_cmd == LPFC_CTX_LUN, the function sends abort only to the * FCP iocbs associated with lun specified by tgt_id and lun_id * parameters @@ -11533,6 +11586,9 @@ lpfc_sli_abort_iocb(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, for (i = 1; i <= phba->sli.last_iotag; i++) { iocbq = phba->sli.iocbq_lookup[i]; + if (lpfc_sli_validate_fcp_iocb_for_abort(iocbq, vport)) + continue; + if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id, abort_cmd) != 0) continue; @@ -11608,7 +11664,11 @@ lpfc_sli_abort_iocb(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, * * This function sends an abort command for every SCSI command * associated with the given virtual port pending on the ring - * filtered by lpfc_sli_validate_fcp_iocb function. + * filtered by lpfc_sli_validate_fcp_iocb_for_abort and then + * lpfc_sli_validate_fcp_iocb function. The ordering for validation before + * submitting abort iocbs must be lpfc_sli_validate_fcp_iocb_for_abort + * followed by lpfc_sli_validate_fcp_iocb. + * * When taskmgmt_cmd == LPFC_CTX_LUN, the function sends abort only to the * FCP iocbs associated with lun specified by tgt_id and lun_id * parameters @@ -11644,6 +11704,9 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, for (i = 1; i <= phba->sli.last_iotag; i++) { iocbq = phba->sli.iocbq_lookup[i]; + if (lpfc_sli_validate_fcp_iocb_for_abort(iocbq, vport)) + continue; + if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id, cmd) != 0) continue; diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 2983e8fb2ec5b9..90582bb5632ba0 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -527,7 +527,7 @@ void nfs_prime_dcache(struct dentry *parent, struct nfs_entry *entry) } } if (!entry->fh->size) - goto out; + return; dentry = d_alloc(parent, &filename); if (dentry == NULL) diff --git a/include/linux/cpu.h b/include/linux/cpu.h index c230cb1c022862..69d391151f2170 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -59,6 +59,8 @@ extern ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, extern ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf); +extern ssize_t cpu_show_retbleed(struct device *dev, + struct device_attribute *attr, char *buf); #ifdef CONFIG_HOTPLUG_CPU extern void unregister_cpu(struct cpu *cpu); diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index bf2b19fddd3bd4..e326524bafccf1 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -511,6 +511,10 @@ struct health_buffer { __be16 ext_synd; }; +enum mlx5_cmd_addr_l_sz_offset { + MLX5_NIC_IFC_OFFSET = 8, +}; + struct mlx5_init_seg { __be32 fw_rev; __be32 cmdif_rev_fw_sub; diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 942ef317d5f51f..4ad4fa403cadff 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -796,6 +796,7 @@ struct mlx5_cmd_work_ent { u64 ts1; u64 ts2; u16 op; + bool polling; }; struct mlx5_pas { @@ -925,6 +926,8 @@ int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int mlx5_cmd_exec_cb(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size, mlx5_cmd_cbk_t callback, void *context); +int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size, + void *out, int out_size); void mlx5_cmd_mbox_status(void *out, u8 *status, u32 *syndrome); int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type); diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index de50303c6d571b..6d308e94b295a0 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -960,13 +960,15 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 log_max_mkey[0x6]; u8 reserved_at_f0[0x8]; u8 dump_fill_mkey[0x1]; - u8 reserved_at_f9[0x3]; + u8 reserved_at_f9[0x2]; + u8 fast_teardown[0x1]; u8 log_max_eq[0x4]; u8 max_indirection[0x8]; u8 fixed_buffer_size[0x1]; u8 log_max_mrw_sz[0x7]; - u8 reserved_at_110[0x2]; + u8 force_teardown[0x1]; + u8 reserved_at_111[0x1]; u8 log_max_bsf_list_size[0x6]; u8 umr_extended_translation_offset[0x1]; u8 null_mkey[0x1]; @@ -3440,18 +3442,26 @@ struct mlx5_ifc_tsar_element_bits { u8 reserved_at_10[0x10]; }; +enum { + MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_SUCCESS = 0x0, + MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL = 0x1, +}; + struct mlx5_ifc_teardown_hca_out_bits { u8 status[0x8]; u8 reserved_at_8[0x18]; u8 syndrome[0x20]; - u8 reserved_at_40[0x40]; + u8 reserved_at_40[0x3f]; + + u8 state[0x1]; }; enum { MLX5_TEARDOWN_HCA_IN_PROFILE_GRACEFUL_CLOSE = 0x0, - MLX5_TEARDOWN_HCA_IN_PROFILE_PANIC_CLOSE = 0x1, + MLX5_TEARDOWN_HCA_IN_PROFILE_FORCE_CLOSE = 0x1, + MLX5_TEARDOWN_HCA_IN_PROFILE_PREPARE_FAST_TEARDOWN = 0x2, }; struct mlx5_ifc_teardown_hca_in_bits { diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index 9aab0941718b16..238aafce0422d8 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c @@ -280,7 +280,8 @@ void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times) /* * Sample a process (thread group) clock for the given group_leader task. - * Must be called with tasklist_lock held for reading. + * Must be called with task sighand lock held for safe while_each_thread() + * traversal. */ static int cpu_clock_sample_group(const clockid_t which_clock, struct task_struct *p, @@ -326,10 +327,8 @@ static int posix_cpu_clock_get(const clockid_t which_clock, struct timespec *tp) error = cpu_clock_sample(which_clock, current, &rtn); } else { - qread_lock(&tasklist_lock); error = cpu_clock_sample_group(which_clock, current, &rtn); - qread_unlock(&tasklist_lock); } } else { /* @@ -417,25 +416,30 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer) static int posix_cpu_timer_del(struct k_itimer *timer) { struct task_struct *p = timer->it.cpu.task; + struct sighand_struct *sighand; + unsigned long flags; int ret = 0; if (likely(p != NULL)) { - qread_lock(&tasklist_lock); - if (unlikely(p->sighand == NULL)) { + /* + * Protect against sighand release/switch in exit/exec and + * process/thread timer list entry concurrent read/writes. + */ + sighand = lock_task_sighand(p, &flags); + if (unlikely(sighand == NULL)) { /* * We raced with the reaping of the task. * The deletion should have cleared us off the list. */ BUG_ON(!list_empty(&timer->it.cpu.entry)); } else { - spin_lock(&p->sighand->siglock); if (timer->it.cpu.firing) ret = TIMER_RETRY; else list_del(&timer->it.cpu.entry); - spin_unlock(&p->sighand->siglock); + + unlock_task_sighand(p, &flags); } - qread_unlock(&tasklist_lock); if (!ret) put_task_struct(p); @@ -534,8 +538,7 @@ static inline int expires_gt(cputime_t expires, cputime_t new_exp) /* * Insert the timer on the appropriate list before any timers that - * expire later. This must be called with the tasklist_lock held - * for reading, interrupts disabled and p->sighand->siglock taken. + * expire later. This must be called with the sighand lock held. */ static void arm_timer(struct k_itimer *timer) { @@ -626,7 +629,8 @@ static void cpu_timer_fire(struct k_itimer *timer) /* * Sample a process (thread group) timer for the given group_leader task. - * Must be called with tasklist_lock held for reading. + * Must be called with task sighand lock held for safe while_each_thread() + * traversal. */ static int cpu_timer_sample_group(const clockid_t which_clock, struct task_struct *p, @@ -689,11 +693,13 @@ static inline void posix_cpu_timer_kick_nohz(void) { } * If we return TIMER_RETRY, it's necessary to release the timer's lock * and try again. (This happens when the timer is in the middle of firing.) */ -static int posix_cpu_timer_set(struct k_itimer *timer, int flags, +static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, struct itimerspec *new, struct itimerspec *old) { struct task_struct *p = timer->it.cpu.task; union cpu_time_count old_expires, new_expires, old_incr, val; + struct sighand_struct *sighand; + unsigned long flags; int ret; if (unlikely(p == NULL)) { @@ -705,14 +711,12 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int flags, new_expires = timespec_to_sample(timer->it_clock, &new->it_value); - qread_lock(&tasklist_lock); /* - * We need the tasklist_lock to protect against reaping that - * clears p->sighand. If p has just been reaped, we can no - * longer get any information about it at all. + * Protect against sighand release/switch in exit/exec and p->cpu_timers + * and p->signal->cpu_timers read/write in arm_timer() */ - if (unlikely(p->sighand == NULL)) { - qread_unlock(&tasklist_lock); + sighand = lock_task_sighand(p, &flags); + if (unlikely(sighand == NULL)) { put_task_struct(p); timer->it.cpu.task = NULL; return -ESRCH; @@ -725,7 +729,6 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int flags, ret = 0; old_incr = timer->it.cpu.incr; - spin_lock(&p->sighand->siglock); old_expires = timer->it.cpu.expires; if (unlikely(timer->it.cpu.firing)) { timer->it.cpu.firing = -1; @@ -785,12 +788,11 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int flags, * disable this firing since we are already reporting * it as an overrun (thanks to bump_cpu_timer above). */ - spin_unlock(&p->sighand->siglock); - qread_unlock(&tasklist_lock); + unlock_task_sighand(p, &flags); goto out; } - if (new_expires.sched != 0 && !(flags & TIMER_ABSTIME)) { + if (new_expires.sched != 0 && !(timer_flags & TIMER_ABSTIME)) { cpu_time_add(timer->it_clock, &new_expires, val); } @@ -805,9 +807,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int flags, arm_timer(timer); } - spin_unlock(&p->sighand->siglock); - qread_unlock(&tasklist_lock); - + unlock_task_sighand(p, &flags); /* * Install the new reload setting, and * set up the signal and overrun bookkeeping. @@ -881,8 +881,16 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp) cpu_clock_sample(timer->it_clock, p, &now); clear_dead = p->exit_state; } else { - qread_lock(&tasklist_lock); - if (unlikely(p->sighand == NULL)) { + struct sighand_struct *sighand; + unsigned long flags; + + /* + * Protect against sighand release/switch in exit/exec and + * also make timer sampling safe if it ends up calling + * thread_group_cputime(). + */ + sighand = lock_task_sighand(p, &flags); + if (unlikely(sighand == NULL)) { /* * The process has been reaped. * We can't even collect a sample any more. @@ -891,14 +899,13 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp) put_task_struct(p); timer->it.cpu.task = NULL; timer->it.cpu.expires.sched = 0; - qread_unlock(&tasklist_lock); goto dead; } else { cpu_timer_sample_group(timer->it_clock, p, &now); clear_dead = (unlikely(p->exit_state) && thread_group_empty(p)); + unlock_task_sighand(p, &flags); } - qread_unlock(&tasklist_lock); } if (unlikely(clear_dead)) { @@ -1175,6 +1182,8 @@ static void check_process_timers(struct task_struct *tsk, void posix_cpu_timer_schedule(struct k_itimer *timer) { struct task_struct *p = timer->it.cpu.task; + struct sighand_struct *sighand; + unsigned long flags; union cpu_time_count now; if (unlikely(p == NULL)) @@ -1193,11 +1202,14 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) clear_dead_task(timer, now); goto out; } - qread_lock(&tasklist_lock); /* arm_timer needs it. */ - spin_lock(&p->sighand->siglock); + sighand = lock_task_sighand(p, &flags); + if (unlikely(sighand == NULL)) { + clear_dead_task(timer, now); + goto out; + } } else { - qread_lock(&tasklist_lock); - if (unlikely(p->sighand == NULL)) { + sighand = lock_task_sighand(p, &flags); + if (unlikely(sighand == NULL)) { /* * The process has been reaped. * We can't even collect a sample any more. @@ -1205,7 +1217,7 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) put_task_struct(p); timer->it.cpu.task = p = NULL; timer->it.cpu.expires.sched = 0; - goto out_unlock; + goto out; } else if (unlikely(p->exit_state) && thread_group_empty(p)) { /* * We've noticed that the thread is dead, but @@ -1215,10 +1227,8 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) clear_dead_task(timer, now); goto out_unlock; } - spin_lock(&p->sighand->siglock); cpu_timer_sample_group(timer->it_clock, p, &now); bump_cpu_timer(timer, now); - /* Leave the tasklist_lock locked for the call below. */ } /* @@ -1226,10 +1236,9 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) */ BUG_ON(!irqs_disabled()); arm_timer(timer); - spin_unlock(&p->sighand->siglock); out_unlock: - qread_unlock(&tasklist_lock); + unlock_task_sighand(p, &flags); out: timer->it_overrun_last = timer->it_overrun; diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c index 2fed29fa504e44..d97c5bcdfa4337 100644 --- a/net/sched/cls_route.c +++ b/net/sched/cls_route.c @@ -526,7 +526,7 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, rcu_assign_pointer(f->next, f1); rcu_assign_pointer(*fp, f); - if (fold && fold->handle && f->handle != fold->handle) { + if (fold) { th = to_hash(fold->handle); h = from_hash(fold->handle >> 16); b = rtnl_dereference(head->table[th]); diff --git a/scripts/kernel.spec b/scripts/kernel.spec index 5cc78237da8100..fc05796efdcb79 100644 --- a/scripts/kernel.spec +++ b/scripts/kernel.spec @@ -20,10 +20,10 @@ Summary: The Linux kernel %global distro_build 1160 %define rpmversion 3.10.0 -%define pkgrelease 1160.76.1.el7 +%define pkgrelease 1160.80.1.el7 # allow pkg_release to have configurable %%{?dist} tag -%define specrelease 1160.76.1%{?dist} +%define specrelease 1160.80.1%{?dist} %define pkg_release %{specrelease}%{?buildid} @@ -1804,6 +1804,66 @@ fi %kernel_variant_files %{with_kdump} kdump %changelog +* Sat Oct 08 2022 Rado Vrbovsky [3.10.0-1160.80.1.el7] +- scsi: lpfc: Fix FCP I/O flush functionality for TMF routines (Dick Kennedy) [1969988] +- scsi: lpfc: Fix illegal memory access on Abort IOCBs (Dick Kennedy) [1969988] +- NFS: Fix extra call to dput() in nfs_prime_dcache (Benjamin Coddington) [2117856] + +* Thu Sep 22 2022 Rado Vrbovsky [3.10.0-1160.79.1.el7] +- x86/speculation: Add LFENCE to RSB fill sequence (Rafael Aquini) [2115073] {CVE-2022-26373} +- x86/speculation: Protect against userspace-userspace spectreRSB (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/speculation: cope with spectre_v2=retpoline cmdline on retbleed-affected Intel CPUs (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- KVM: emulate: do not adjust size of fastop and setcc subroutines (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/kvm: fix FASTOP_SIZE when return thunks are enabled (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/speculation: Disable RRSBA behavior (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/kexec: Disable RET on kexec (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/bugs: Do not enable IBPB-on-entry when IBPB is not supported (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/bugs: Add Cannon lake to RETBleed affected CPU list (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/cpu/amd: Enumerate BTC_NO (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/common: Stamp out the stepping madness (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/cpu/amd: Add Spectral Chicken (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/bugs: Do IBPB fallback check only once (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/bugs: Add retbleed=ibpb (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/bugs: Report Intel retbleed vulnerability (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/bugs: Enable STIBP for JMP2RET (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/bugs: Add AMD retbleed= boot parameter (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/bugs: Report AMD retbleed vulnerability (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86: Add magic AMD return-thunk (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86: Use return-thunk in asm code (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/sev: Avoid using __x86_return_thunk (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/vsyscall_emu/64: Don't use RET in vsyscall emulation (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/kvm: Fix SETcc emulation for return thunks (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86,objtool: Create .return_sites (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86: Undo return-thunk damage (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/retpoline: Use -mfunction-return (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/cpufeatures: Move RETPOLINE flags to word 11 (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- objtool: Add ELF writing capability (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86: Prepare asm files for straight-line-speculation (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86: Prepare inline-asm for straight-line-speculation (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/kvm: Fix fastop function ELF metadata (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/kvm: Move kvm_fastop_exception to .fixup section (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/vdso: Fix vDSO build if a retpoline is emitted (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/cpufeatures: Combine word 11 and 12 into a new scattered features word (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/cpufeatures: Carve out CQM features retrieval (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/cpufeatures: Re-tabulate the X86_FEATURE definitions (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/cpufeature: Move processor tracing out of scattered features (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/cpu: Probe CPUID leaf 6 even when cpuid_level == 6 (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} +- x86/alternatives: Cleanup DPRINTK macro (Rafael Aquini) [2090227] {CVE-2022-23816 CVE-2022-23825 CVE-2022-29900 CVE-2022-29901} + +* Thu Sep 15 2022 Rado Vrbovsky [3.10.0-1160.78.1.el7] +- net_sched: cls_route: remove from list when handle is 0 (Davide Caratti) [2121809] {CVE-2022-2588} + +* Thu Sep 08 2022 Rado Vrbovsky [3.10.0-1160.77.1.el7] +- net/mlx5: Add Fast teardown support (Jay Shin) [2077711] +- net/mlx5: Free IRQs in shutdown path (Jay Shin) [2077711] +- net/mlx5: Change teardown with force mode failure message to warning (Jay Shin) [2077711] +- net/mlx5: Cancel health poll before sending panic teardown command (Jay Shin) [2077711] +- net/mlx5: Add fast unload support in shutdown flow (Jay Shin) [2077711] +- net/mlx5: Expose command polling interface (Jay Shin) [2077711] +- posix-timers: Remove remaining uses of tasklist_lock (Oleg Nesterov) [2115147] +- posix-timers: Use sighand lock instead of tasklist_lock on timer deletion (Oleg Nesterov) [2115147] +- posix-cpu-timers: remove tasklist_lock in posix_cpu_clock_get() (Oleg Nesterov) [2115147] + * Tue Jul 26 2022 Rado Vrbovsky [3.10.0-1160.76.1.el7] - sfc: complete the next packet when we receive a timestamp (Íñigo Huguet) [1793280] diff --git a/tools/objtool/arch.h b/tools/objtool/arch.h index f7350fcedc70dc..7707b3bb359ef9 100644 --- a/tools/objtool/arch.h +++ b/tools/objtool/arch.h @@ -41,4 +41,6 @@ int arch_decode_instruction(struct elf *elf, struct section *sec, unsigned int *len, unsigned char *type, unsigned long *displacement); +bool arch_is_rethunk(struct symbol *sym); + #endif /* _ARCH_H */ diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c index c0c0b265e88e54..85312190fa2159 100644 --- a/tools/objtool/arch/x86/decode.c +++ b/tools/objtool/arch/x86/decode.c @@ -53,7 +53,7 @@ int arch_decode_instruction(struct elf *elf, struct section *sec, if (x86_64 == -1) return -1; - insn_init(&insn, (void *)(sec->data + offset), maxlen, x86_64); + insn_init(&insn, (void *)(sec->data->d_buf + offset), maxlen, x86_64); insn_get_length(&insn); insn_get_opcode(&insn); insn_get_modrm(&insn); @@ -170,3 +170,8 @@ int arch_decode_instruction(struct elf *elf, struct section *sec, return 0; } + +bool arch_is_rethunk(struct symbol *sym) +{ + return !strcmp(sym->name, "__x86_return_thunk"); +} diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c index c7a4d2891eb181..26b74b47182a98 100644 --- a/tools/objtool/builtin-check.c +++ b/tools/objtool/builtin-check.c @@ -25,6 +25,7 @@ * For more information, see tools/objtool/Documentation/stack-validation.txt. */ +#include #include #include #include @@ -45,6 +46,7 @@ struct instruction { struct list_head list; struct hlist_node hash; + struct list_head call_node; struct section *sec; unsigned long offset; unsigned int len, state; @@ -66,6 +68,7 @@ struct objtool_file { struct elf *elf; struct list_head insn_list; DECLARE_HASHTABLE(insn_hash, 16); + struct list_head return_thunk_list; struct section *rodata, *whitelist; bool ignore_unreachables, c_file; }; @@ -271,6 +274,8 @@ static int decode_instructions(struct objtool_file *file) memset(insn, 0, sizeof(*insn)); INIT_LIST_HEAD(&insn->alts); + INIT_LIST_HEAD(&insn->call_node); + insn->sec = sec; insn->offset = offset; @@ -310,6 +315,67 @@ static int decode_instructions(struct objtool_file *file) return 0; } +static int create_return_sites_sections(struct objtool_file *file) +{ + struct instruction *insn; + struct section *sec, *rela_sec; + struct rela *rela; + int idx; + + sec = find_section_by_name(file->elf, ".return_sites"); + if (sec) { + WARN("file already has .return_sites, skipping"); + return 0; + } + + idx = 0; + list_for_each_entry(insn, &file->return_thunk_list, call_node) + idx++; + + if (!idx) + return 0; + + sec = elf_create_section(file->elf, ".return_sites", + sizeof(int), idx); + if (!sec) { + WARN("elf_create_section: .return_sites"); + return -1; + } + + rela_sec = elf_create_rela_section(file->elf, sec); + if (!rela_sec) + return -1; + + idx = 0; + list_for_each_entry(insn, &file->return_thunk_list, call_node) { + + int *site = (int *)sec->data->d_buf + idx; + *site = 0; + + rela = malloc(sizeof(*rela)); + if (!rela) { + perror("malloc"); + return -1; + } + memset(rela, 0, sizeof(*rela)); + + rela->sym = insn->sec->sym; + rela->addend = insn->offset; + rela->type = R_X86_64_PC32; + rela->offset = idx * sizeof(*site); + + list_add_tail(&rela->list, &rela_sec->rela_list); + hash_add(rela_sec->rela_hash, &rela->hash, rela->offset); + + idx++; + } + + if (elf_rebuild_rela_section(rela_sec)) + return -1; + + return 0; +} + /* * Warnings shouldn't be reported for ignored functions. */ @@ -333,6 +399,28 @@ static void add_ignores(struct objtool_file *file) } } +__weak bool arch_is_rethunk(struct symbol *sym) +{ + return false; +} + +static void add_return_call(struct objtool_file *file, struct instruction *insn) +{ + /* + * Return thunk tail calls are really just returns in disguise, + * so convert them accordingly. + */ + insn->type = INSN_RETURN; + + /* + * Do not add instructions that are in .discard.text section + * to avoid the "'.discard.text' referenced in section `.return_sites' + * of arch/x86/built-in.o" linker error + */ + if (!!strncmp(insn->sec->name, ".discard", 8)) + list_add_tail(&insn->call_node, &file->return_thunk_list); +} + /* * Find the destination instructions for all jumps. */ @@ -370,6 +458,9 @@ static int add_jump_destinations(struct objtool_file *file) */ insn->type = INSN_JUMP_DYNAMIC; continue; + } else if (arch_is_rethunk(rela->sym)) { + add_return_call(file, insn); + continue; } else { /* sibling call */ insn->jump_dest = 0; @@ -1214,7 +1305,7 @@ int cmd_check(int argc, const char **argv) objname = argv[0]; - file.elf = elf_open(objname); + file.elf = elf_open(objname, O_RDWR); if (!file.elf) { fprintf(stderr, "error reading elf file %s\n", objname); return 1; @@ -1222,6 +1313,7 @@ int cmd_check(int argc, const char **argv) INIT_LIST_HEAD(&file.insn_list); hash_init(file.insn_hash); + INIT_LIST_HEAD(&file.return_thunk_list); file.whitelist = find_section_by_name(file.elf, "__func_stack_frame_non_standard"); file.rodata = find_section_by_name(file.elf, ".rodata"); file.ignore_unreachables = false; @@ -1237,11 +1329,20 @@ int cmd_check(int argc, const char **argv) goto out; warnings += ret; + ret = create_return_sites_sections(&file); + if (ret < 0) + goto out; + warnings += ret; + ret = validate_uncallable_instructions(&file); if (ret < 0) goto out; warnings += ret; + ret = elf_write(file.elf); + if (ret < 0) + goto out; + out: cleanup(&file); diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index e11f6b69cce67e..e8b1d695e1a276 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -163,20 +163,19 @@ static int read_sections(struct elf *elf) return -1; } - sec->elf_data = elf_getdata(s, NULL); - if (!sec->elf_data) { + sec->data = elf_getdata(s, NULL); + if (!sec->data) { perror("elf_getdata"); return -1; } - if (sec->elf_data->d_off != 0 || - sec->elf_data->d_size != sec->sh.sh_size) { + if (sec->data->d_off != 0 || + sec->data->d_size != sec->sh.sh_size) { WARN("unexpected data attributes for %s", sec->name); return -1; } - sec->data = (unsigned long)sec->elf_data->d_buf; - sec->len = sec->elf_data->d_size; + sec->len = sec->data->d_size; } /* sanity check, one more call to elf_nextscn() should return NULL */ @@ -213,7 +212,7 @@ static int read_symbols(struct elf *elf) sym->idx = i; - if (!gelf_getsym(symtab->elf_data, i, &sym->sym)) { + if (!gelf_getsym(symtab->data, i, &sym->sym)) { perror("gelf_getsym"); goto err; } @@ -303,7 +302,7 @@ static int read_relas(struct elf *elf) } memset(rela, 0, sizeof(*rela)); - if (!gelf_getrela(sec->elf_data, i, &rela->rela)) { + if (!gelf_getrela(sec->data, i, &rela->rela)) { perror("gelf_getrela"); return -1; } @@ -328,9 +327,10 @@ static int read_relas(struct elf *elf) return 0; } -struct elf *elf_open(const char *name) +struct elf *elf_open(const char *name, int flags) { struct elf *elf; + Elf_Cmd cmd; elf_version(EV_CURRENT); @@ -349,13 +349,20 @@ struct elf *elf_open(const char *name) goto err; } - elf->fd = open(name, O_RDONLY); + elf->fd = open(name, flags); if (elf->fd == -1) { perror("open"); goto err; } - elf->elf = elf_begin(elf->fd, ELF_C_READ_MMAP, NULL); + if ((flags & O_ACCMODE) == O_RDONLY) + cmd = ELF_C_READ_MMAP; + else if ((flags & O_ACCMODE) == O_RDWR) + cmd = ELF_C_RDWR; + else /* O_WRONLY */ + cmd = ELF_C_WRITE; + + elf->elf = elf_begin(elf->fd, cmd, NULL); if (!elf->elf) { perror("elf_begin"); goto err; @@ -382,6 +389,194 @@ struct elf *elf_open(const char *name) return NULL; } +struct section *elf_create_section(struct elf *elf, const char *name, + size_t entsize, int nr) +{ + struct section *sec, *shstrtab; + size_t size = entsize * nr; + struct Elf_Scn *s; + Elf_Data *data; + + sec = malloc(sizeof(*sec)); + if (!sec) { + perror("malloc"); + return NULL; + } + memset(sec, 0, sizeof(*sec)); + + INIT_LIST_HEAD(&sec->symbol_list); + INIT_LIST_HEAD(&sec->rela_list); + hash_init(sec->rela_hash); + hash_init(sec->symbol_hash); + + list_add_tail(&sec->list, &elf->sections); + + s = elf_newscn(elf->elf); + if (!s) { + perror("elf_newscn"); + return NULL; + } + + sec->name = strdup(name); + if (!sec->name) { + perror("strdup"); + return NULL; + } + + sec->idx = elf_ndxscn(s); + sec->len = size; + sec->changed = true; + + sec->data = elf_newdata(s); + if (!sec->data) { + perror("elf_newdata"); + return NULL; + } + + sec->data->d_size = size; + sec->data->d_align = 1; + + if (size) { + sec->data->d_buf = malloc(size); + if (!sec->data->d_buf) { + perror("malloc"); + return NULL; + } + memset(sec->data->d_buf, 0, size); + } + + if (!gelf_getshdr(s, &sec->sh)) { + perror("gelf_getshdr"); + return NULL; + } + + sec->sh.sh_size = size; + sec->sh.sh_entsize = entsize; + sec->sh.sh_type = SHT_PROGBITS; + sec->sh.sh_addralign = 1; + sec->sh.sh_flags = SHF_ALLOC; + + + /* Add section name to .shstrtab */ + shstrtab = find_section_by_name(elf, ".shstrtab"); + if (!shstrtab) { + WARN("can't find .shstrtab section"); + return NULL; + } + + s = elf_getscn(elf->elf, shstrtab->idx); + if (!s) { + perror("elf_getscn"); + return NULL; + } + + data = elf_newdata(s); + if (!data) { + perror("elf_newdata"); + return NULL; + } + + data->d_buf = sec->name; + data->d_size = strlen(name) + 1; + data->d_align = 1; + + sec->sh.sh_name = shstrtab->len; + + shstrtab->len += strlen(name) + 1; + shstrtab->changed = true; + + return sec; +} + +struct section *elf_create_rela_section(struct elf *elf, struct section *base) +{ + char *relaname; + struct section *sec; + + relaname = malloc(strlen(base->name) + strlen(".rela") + 1); + if (!relaname) { + perror("malloc"); + return NULL; + } + strcpy(relaname, ".rela"); + strcat(relaname, base->name); + + sec = elf_create_section(elf, relaname, sizeof(GElf_Rela), 0); + if (!sec) + return NULL; + + base->rela = sec; + sec->base = base; + + sec->sh.sh_type = SHT_RELA; + sec->sh.sh_addralign = 8; + sec->sh.sh_link = find_section_by_name(elf, ".symtab")->idx; + sec->sh.sh_info = base->idx; + sec->sh.sh_flags = SHF_INFO_LINK; + + return sec; +} + +int elf_rebuild_rela_section(struct section *sec) +{ + struct rela *rela; + int nr, idx = 0, size; + GElf_Rela *relas; + + nr = 0; + list_for_each_entry(rela, &sec->rela_list, list) + nr++; + + size = nr * sizeof(*relas); + relas = malloc(size); + if (!relas) { + perror("malloc"); + return -1; + } + + sec->data->d_buf = relas; + sec->data->d_size = size; + + sec->sh.sh_size = size; + + idx = 0; + list_for_each_entry(rela, &sec->rela_list, list) { + relas[idx].r_offset = rela->offset; + relas[idx].r_addend = rela->addend; + relas[idx].r_info = GELF_R_INFO(rela->sym->idx, rela->type); + idx++; + } + + return 0; +} + +int elf_write(struct elf *elf) +{ + struct section *sec; + Elf_Scn *s; + + list_for_each_entry(sec, &elf->sections, list) { + if (sec->changed) { + s = elf_getscn(elf->elf, sec->idx); + if (!s) { + perror("elf_getscn"); + return -1; + } + if (!gelf_update_shdr (s, &sec->sh)) { + perror("gelf_update_shdr"); + return -1; + } + } + } + + if (elf_update(elf->elf, ELF_C_WRITE) < 0) { + perror("elf_update"); + return -1; + } + + return 0; +} + void elf_close(struct elf *elf) { struct section *sec, *tmpsec; diff --git a/tools/objtool/elf.h b/tools/objtool/elf.h index aa1ff6596684f9..2f31c86b50c99c 100644 --- a/tools/objtool/elf.h +++ b/tools/objtool/elf.h @@ -28,6 +28,13 @@ # define elf_getshdrstrndx elf_getshstrndx #endif +/* + * Fallback for systems without this "read, mmaping if possible" cmd. + */ +#ifndef ELF_C_READ_MMAP +#define ELF_C_READ_MMAP ELF_C_READ +#endif + struct section { struct list_head list; GElf_Shdr sh; @@ -37,11 +44,11 @@ struct section { DECLARE_HASHTABLE(rela_hash, 16); struct section *base, *rela; struct symbol *sym; - Elf_Data *elf_data; + Elf_Data *data; char *name; int idx; - unsigned long data; unsigned int len; + bool changed, text; }; struct symbol { @@ -54,6 +61,7 @@ struct symbol { unsigned char bind, type; unsigned long offset; unsigned int len; + bool return_thunk; }; struct rela { @@ -76,13 +84,18 @@ struct elf { }; -struct elf *elf_open(const char *name); +struct elf *elf_open(const char *name, int flags); struct section *find_section_by_name(struct elf *elf, const char *name); struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset); struct rela *find_rela_by_dest(struct section *sec, unsigned long offset); struct rela *find_rela_by_dest_range(struct section *sec, unsigned long offset, unsigned int len); struct symbol *find_containing_func(struct section *sec, unsigned long offset); +struct section *elf_create_section(struct elf *elf, const char *name, size_t + entsize, int nr); +struct section *elf_create_rela_section(struct elf *elf, struct section *base); +int elf_rebuild_rela_section(struct section *sec); +int elf_write(struct elf *elf); void elf_close(struct elf *elf); diff --git a/tools/objtool/special.c b/tools/objtool/special.c index 89496f93670d53..17ca798218a2f5 100644 --- a/tools/objtool/special.c +++ b/tools/objtool/special.c @@ -91,16 +91,16 @@ static int get_alt_entry(struct elf *elf, struct special_entry *entry, alt->jump_or_nop = entry->jump_or_nop; if (alt->group) { - alt->orig_len = *(unsigned char *)(sec->data + offset + + alt->orig_len = *(unsigned char *)(sec->data->d_buf + offset + entry->orig_len); - alt->new_len = *(unsigned char *)(sec->data + offset + + alt->new_len = *(unsigned char *)(sec->data->d_buf + offset + entry->new_len); } if (entry->feature) { unsigned short feature; - feature = *(unsigned short *)(sec->data + offset + + feature = *(unsigned short *)(sec->data->d_buf + offset + entry->feature); /*