From dd4356a4adeef5b48298d2fcce2b8fcc668d9d4b Mon Sep 17 00:00:00 2001 From: Bill Yang Date: Mon, 31 Jul 2023 14:39:13 -0700 Subject: [PATCH 1/4] Add BoringSSL Dispatch Test for aarch64 --- crypto/fipsmodule/aes/asm/aesv8-armx.pl | 25 ++++++ crypto/fipsmodule/aes/asm/vpaes-armv8.pl | 14 ++++ crypto/fipsmodule/sha/asm/sha512-armv8.pl | 14 ++++ crypto/impl_dispatch_test.cc | 82 +++++++++++++------ crypto/internal.h | 9 ++ .../crypto/fipsmodule/aesv8-armx.S | 28 +++++++ .../crypto/fipsmodule/sha256-armv8.S | 7 ++ .../crypto/fipsmodule/sha512-armv8.S | 7 ++ .../crypto/fipsmodule/vpaes-armv8.S | 14 ++++ .../crypto/fipsmodule/aesv8-armx.S | 28 +++++++ .../crypto/fipsmodule/sha256-armv8.S | 7 ++ .../crypto/fipsmodule/sha512-armv8.S | 7 ++ .../crypto/fipsmodule/vpaes-armv8.S | 14 ++++ .../crypto/fipsmodule/aesv8-armx.S | 28 +++++++ .../crypto/fipsmodule/sha256-armv8.S | 7 ++ .../crypto/fipsmodule/sha512-armv8.S | 7 ++ .../crypto/fipsmodule/vpaes-armv8.S | 14 ++++ 17 files changed, 289 insertions(+), 23 deletions(-) diff --git a/crypto/fipsmodule/aes/asm/aesv8-armx.pl b/crypto/fipsmodule/aes/asm/aesv8-armx.pl index 9f62232e93..76f4698415 100644 --- a/crypto/fipsmodule/aes/asm/aesv8-armx.pl +++ b/crypto/fipsmodule/aes/asm/aesv8-armx.pl @@ -96,6 +96,13 @@ .Lenc_key: ___ $code.=<<___ if ($flavour =~ /64/); +#ifdef BORINGSSL_DISPATCH_TEST +.extern BORINGSSL_function_hit + adrp x6,:pg_hi21:BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#3] // kFlag_aes_hw_set_encrypt_key +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! @@ -340,6 +347,17 @@ () .type ${prefix}_${dir}crypt,%function .align 5 ${prefix}_${dir}crypt: +___ +$code.=<<___ if ($flavour =~ /64/); +#ifdef BORINGSSL_DISPATCH_TEST +.extern BORINGSSL_function_hit + adrp x6,:pg_hi21:BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#1] // kFlag_aes_hw_encrypt +#endif +___ +$code.=<<___; AARCH64_VALID_CALL_TARGET ldr $rounds,[$key,#240] vld1.32 {$rndkey0},[$key],#16 @@ -719,6 +737,13 @@ () ${prefix}_ctr32_encrypt_blocks: ___ $code.=<<___ if ($flavour =~ /64/); +#ifdef BORINGSSL_DISPATCH_TEST +.extern BORINGSSL_function_hit + adrp x6,:pg_hi21:BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6] // kFlag_aes_hw_ctr32_encrypt_blocks +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! diff --git a/crypto/fipsmodule/aes/asm/vpaes-armv8.pl b/crypto/fipsmodule/aes/asm/vpaes-armv8.pl index 8b01fed871..413633a29c 100755 --- a/crypto/fipsmodule/aes/asm/vpaes-armv8.pl +++ b/crypto/fipsmodule/aes/asm/vpaes-armv8.pl @@ -261,6 +261,13 @@ .type vpaes_encrypt,%function .align 4 vpaes_encrypt: +#ifdef BORINGSSL_DISPATCH_TEST +.extern BORINGSSL_function_hit + adrp x6,:pg_hi21:BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#4] // kFlag_vpaes_encrypt +#endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1069,6 +1076,13 @@ .type vpaes_set_encrypt_key,%function .align 4 vpaes_set_encrypt_key: +#ifdef BORINGSSL_DISPATCH_TEST +.extern BORINGSSL_function_hit + adrp x6,:pg_hi21:BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#5] // kFlag_vpaes_set_encrypt_key +#endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/crypto/fipsmodule/sha/asm/sha512-armv8.pl b/crypto/fipsmodule/sha/asm/sha512-armv8.pl index 8cb312facc..100bcfe549 100644 --- a/crypto/fipsmodule/sha/asm/sha512-armv8.pl +++ b/crypto/fipsmodule/sha/asm/sha512-armv8.pl @@ -359,6 +359,13 @@ sub BODY_00_xx { .align 6 sha256_block_armv8: .Lv8_entry: +#ifdef BORINGSSL_DISPATCH_TEST +.extern BORINGSSL_function_hit + adrp x6,:pg_hi21:BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#6] // kFlag_sha256_hw +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -445,6 +452,13 @@ sub BODY_00_xx { .align 6 sha512_block_armv8: .Lv8_entry: +#ifdef BORINGSSL_DISPATCH_TEST +.extern BORINGSSL_function_hit + adrp x6,:pg_hi21:BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#2] // kFlag_sha512_hw +#endif stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/crypto/impl_dispatch_test.cc b/crypto/impl_dispatch_test.cc index f31c687955..a4ff1bcc7f 100644 --- a/crypto/impl_dispatch_test.cc +++ b/crypto/impl_dispatch_test.cc @@ -29,14 +29,19 @@ #include "internal.h" +#if defined(OPENSSL_AARCH64) +#include +extern uint32_t OPENSSL_armcap_P; +#endif + class ImplDispatchTest : public ::testing::Test { public: void SetUp() override { #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) - aesni_ = OPENSSL_ia32cap_P[1] & (1 << (57 - 32)); + aes_hw_ = OPENSSL_ia32cap_P[1] & (1 << (57 - 32)); avx_movbe_ = ((OPENSSL_ia32cap_P[1] >> 22) & 0x41) == 0x41; - ssse3_ = OPENSSL_ia32cap_P[1] & (1 << (41 - 32)); + aes_vpaes_ = OPENSSL_ia32cap_P[1] & (1 << (41 - 32)); sha_ext_ = OPENSSL_ia32cap_P[2] & (1 << 29); is_x86_64_ = #if defined(OPENSSL_X86_64) @@ -44,7 +49,12 @@ class ImplDispatchTest : public ::testing::Test { #else false; #endif -#endif // X86 || X86_64 +#elif defined(OPENSSL_AARCH64) + aes_hw_ = CRYPTO_is_ARMv8_AES_capable(); + aes_vpaes_ = CRYPTO_is_NEON_capable(); + sha_ext_ = OPENSSL_armcap_P & ARMV8_SHA256; + sha_512_ext_ = OPENSSL_armcap_P & ARMV8_SHA512; +#endif } protected: @@ -72,39 +82,51 @@ class ImplDispatchTest : public ::testing::Test { } } + bool aes_hw_ = false; + bool aes_vpaes_ = false; + bool sha_ext_ = false; #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) - bool aesni_ = false; bool avx_movbe_ = false; - bool ssse3_ = false; - bool sha_ext_ = false; bool is_x86_64_ = false; +#else // AARCH64 + bool sha_512_ext_ = false; #endif + }; -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86) || defined(OPENSSL_X86_64)) +#if !defined(OPENSSL_NO_ASM) && (defined(OPENSSL_X86) || \ + defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) constexpr size_t kFlag_aes_hw_ctr32_encrypt_blocks = 0; constexpr size_t kFlag_aes_hw_encrypt = 1; -constexpr size_t kFlag_aesni_gcm_encrypt = 2; constexpr size_t kFlag_aes_hw_set_encrypt_key = 3; constexpr size_t kFlag_vpaes_encrypt = 4; constexpr size_t kFlag_vpaes_set_encrypt_key = 5; -constexpr size_t kFlag_sha256_shaext = 6; +constexpr size_t kFlag_sha256_hw = 6; +#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) +constexpr size_t kFlag_aesni_gcm_encrypt = 2; +#else // AARCH64 +constexpr size_t kFlag_sha512_hw = 2; +#endif TEST_F(ImplDispatchTest, AEAD_AES_GCM) { AssertFunctionsHit( { - {kFlag_aes_hw_ctr32_encrypt_blocks, aesni_}, - {kFlag_aes_hw_encrypt, aesni_}, - {kFlag_aes_hw_set_encrypt_key, aesni_}, - {kFlag_aesni_gcm_encrypt, is_x86_64_ && aesni_ && avx_movbe_}, - {kFlag_vpaes_encrypt, ssse3_ && !aesni_}, - {kFlag_vpaes_set_encrypt_key, ssse3_ && !aesni_}, + {kFlag_aes_hw_encrypt, aes_hw_}, + {kFlag_aes_hw_set_encrypt_key, aes_hw_}, + {kFlag_vpaes_encrypt, aes_vpaes_ && !aes_hw_}, + {kFlag_vpaes_set_encrypt_key, aes_vpaes_ && !aes_hw_}, +#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) + {kFlag_aes_hw_ctr32_encrypt_blocks, aes_hw_}, + {kFlag_aesni_gcm_encrypt, + is_x86_64_ && aes_hw_ && avx_movbe_}, +#else // AARCH64 + {kFlag_aes_hw_ctr32_encrypt_blocks, aes_hw_} +#endif }, [] { const uint8_t kZeros[16] = {0}; - const uint8_t kPlaintext[40] = {1, 2, 3, 4, 0}; + const uint8_t kPlaintext[256] = {1, 2, 3, 4, 0}; uint8_t ciphertext[sizeof(kPlaintext) + 16]; size_t ciphertext_len; bssl::ScopedEVP_AEAD_CTX ctx; @@ -121,8 +143,8 @@ TEST_F(ImplDispatchTest, AEAD_AES_GCM) { TEST_F(ImplDispatchTest, AES_set_encrypt_key) { AssertFunctionsHit( { - {kFlag_aes_hw_set_encrypt_key, aesni_}, - {kFlag_vpaes_set_encrypt_key, ssse3_ && !aesni_}, + {kFlag_aes_hw_set_encrypt_key, aes_hw_}, + {kFlag_vpaes_set_encrypt_key, aes_vpaes_ && !aes_hw_}, }, [] { AES_KEY key; @@ -138,8 +160,8 @@ TEST_F(ImplDispatchTest, AES_single_block) { AssertFunctionsHit( { - {kFlag_aes_hw_encrypt, aesni_}, - {kFlag_vpaes_encrypt, ssse3_ && !aesni_}, + {kFlag_aes_hw_encrypt, aes_hw_}, + {kFlag_vpaes_encrypt, aes_vpaes_ && !aes_hw_}, }, [&key] { uint8_t in[AES_BLOCK_SIZE] = {0}; @@ -151,7 +173,7 @@ TEST_F(ImplDispatchTest, AES_single_block) { TEST_F(ImplDispatchTest, SHA256) { AssertFunctionsHit( { - {kFlag_sha256_shaext, sha_ext_}, + {kFlag_sha256_hw, sha_ext_}, }, [] { const uint8_t in[32] = {0}; @@ -160,6 +182,20 @@ TEST_F(ImplDispatchTest, SHA256) { }); } -#endif // X86 || X86_64 +#ifdef OPENSSL_AARCH64 +TEST_F(ImplDispatchTest, SHA512) { + AssertFunctionsHit( + { + {kFlag_sha512_hw, sha_512_ext_}, + }, + [] { + const uint8_t in[32] = {0}; + uint8_t out[SHA512_DIGEST_LENGTH]; + SHA512(in, 32, out); + }); +} +#endif // OPENSSL_AARCH64 + +#endif // !OPENSSL_NO_ASM && (OPENSSL_X86 || OPENSSL_X86_64 || OPENSSL_AARCH64) #endif // DISPATCH_TEST && !SHARED_LIBRARY diff --git a/crypto/internal.h b/crypto/internal.h index e462177cef..4f15a7232f 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -1055,6 +1055,7 @@ extern unsigned long OPENSSL_ppc64le_hwcap2; // BORINGSSL_function_hit is an array of flags. The following functions will // set these flags if BORINGSSL_DISPATCH_TEST is defined. +// On x86 and x86_64: // 0: aes_hw_ctr32_encrypt_blocks // 1: aes_hw_encrypt // 2: aesni_gcm_encrypt @@ -1062,6 +1063,14 @@ extern unsigned long OPENSSL_ppc64le_hwcap2; // 4: vpaes_encrypt // 5: vpaes_set_encrypt_key // 6: sha256_block_data_order_shaext +// On AARCH64: +// 0: aes_hw_ctr32_encrypt_blocks +// 1: aes_hw_encrypt +// 2: sha512_block_armv8 +// 3: aes_hw_set_encrypt_key +// 4: vpaes_encrypt +// 5: vpaes_set_encrypt_key +// 6: sha256_block_armv8 extern uint8_t BORINGSSL_function_hit[7]; #endif // BORINGSSL_DISPATCH_TEST diff --git a/generated-src/ios-aarch64/crypto/fipsmodule/aesv8-armx.S b/generated-src/ios-aarch64/crypto/fipsmodule/aesv8-armx.S index 50d7deaa5c..9ad7ff41f5 100644 --- a/generated-src/ios-aarch64/crypto/fipsmodule/aesv8-armx.S +++ b/generated-src/ios-aarch64/crypto/fipsmodule/aesv8-armx.S @@ -32,6 +32,13 @@ Lrcon: .align 5 _aes_hw_set_encrypt_key: Lenc_key: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,_BORINGSSL_function_hit@PAGE + add x6, x6, _BORINGSSL_function_hit@PAGEOFF + mov w7, #1 + strb w7, [x6,#3] // kFlag_aes_hw_set_encrypt_key +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! @@ -244,6 +251,13 @@ Ldec_key_abort: .align 5 _aes_hw_encrypt: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,_BORINGSSL_function_hit@PAGE + add x6, x6, _BORINGSSL_function_hit@PAGEOFF + mov w7, #1 + strb w7, [x6,#1] // kFlag_aes_hw_encrypt +#endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 @@ -275,6 +289,13 @@ Loop_enc: .align 5 _aes_hw_decrypt: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,_BORINGSSL_function_hit@PAGE + add x6, x6, _BORINGSSL_function_hit@PAGEOFF + mov w7, #1 + strb w7, [x6,#1] // kFlag_aes_hw_encrypt +#endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 @@ -599,6 +620,13 @@ Lcbc_abort: .align 5 _aes_hw_ctr32_encrypt_blocks: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,_BORINGSSL_function_hit@PAGE + add x6, x6, _BORINGSSL_function_hit@PAGEOFF + mov w7, #1 + strb w7, [x6] // kFlag_aes_hw_ctr32_encrypt_blocks +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! diff --git a/generated-src/ios-aarch64/crypto/fipsmodule/sha256-armv8.S b/generated-src/ios-aarch64/crypto/fipsmodule/sha256-armv8.S index b40b260fde..acb4990a0d 100644 --- a/generated-src/ios-aarch64/crypto/fipsmodule/sha256-armv8.S +++ b/generated-src/ios-aarch64/crypto/fipsmodule/sha256-armv8.S @@ -1072,6 +1072,13 @@ LK256: .align 6 sha256_block_armv8: Lv8_entry: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,_BORINGSSL_function_hit@PAGE + add x6, x6, _BORINGSSL_function_hit@PAGEOFF + mov w7, #1 + strb w7, [x6,#6] // kFlag_sha256_hw +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/ios-aarch64/crypto/fipsmodule/sha512-armv8.S b/generated-src/ios-aarch64/crypto/fipsmodule/sha512-armv8.S index b2d366d781..45621ddfbd 100644 --- a/generated-src/ios-aarch64/crypto/fipsmodule/sha512-armv8.S +++ b/generated-src/ios-aarch64/crypto/fipsmodule/sha512-armv8.S @@ -1096,6 +1096,13 @@ LK512: .align 6 sha512_block_armv8: Lv8_entry: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,_BORINGSSL_function_hit@PAGE + add x6, x6, _BORINGSSL_function_hit@PAGEOFF + mov w7, #1 + strb w7, [x6,#2] // kFlag_sha512_hw +#endif stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/ios-aarch64/crypto/fipsmodule/vpaes-armv8.S b/generated-src/ios-aarch64/crypto/fipsmodule/vpaes-armv8.S index 5985c55039..096dff99d2 100644 --- a/generated-src/ios-aarch64/crypto/fipsmodule/vpaes-armv8.S +++ b/generated-src/ios-aarch64/crypto/fipsmodule/vpaes-armv8.S @@ -216,6 +216,13 @@ Lenc_entry: .align 4 _vpaes_encrypt: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,_BORINGSSL_function_hit@PAGE + add x6, x6, _BORINGSSL_function_hit@PAGEOFF + mov w7, #1 + strb w7, [x6,#4] // kFlag_vpaes_encrypt +#endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1019,6 +1026,13 @@ Lschedule_mangle_both: .align 4 _vpaes_set_encrypt_key: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,_BORINGSSL_function_hit@PAGE + add x6, x6, _BORINGSSL_function_hit@PAGEOFF + mov w7, #1 + strb w7, [x6,#5] // kFlag_vpaes_set_encrypt_key +#endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/linux-aarch64/crypto/fipsmodule/aesv8-armx.S b/generated-src/linux-aarch64/crypto/fipsmodule/aesv8-armx.S index 4949ba30e6..a8cdd2acab 100644 --- a/generated-src/linux-aarch64/crypto/fipsmodule/aesv8-armx.S +++ b/generated-src/linux-aarch64/crypto/fipsmodule/aesv8-armx.S @@ -33,6 +33,13 @@ .align 5 aes_hw_set_encrypt_key: .Lenc_key: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#3] // kFlag_aes_hw_set_encrypt_key +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! @@ -245,6 +252,13 @@ aes_hw_set_decrypt_key: .type aes_hw_encrypt,%function .align 5 aes_hw_encrypt: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#1] // kFlag_aes_hw_encrypt +#endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 @@ -276,6 +290,13 @@ aes_hw_encrypt: .type aes_hw_decrypt,%function .align 5 aes_hw_decrypt: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#1] // kFlag_aes_hw_encrypt +#endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 @@ -600,6 +621,13 @@ aes_hw_cbc_encrypt: .type aes_hw_ctr32_encrypt_blocks,%function .align 5 aes_hw_ctr32_encrypt_blocks: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6] // kFlag_aes_hw_ctr32_encrypt_blocks +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! diff --git a/generated-src/linux-aarch64/crypto/fipsmodule/sha256-armv8.S b/generated-src/linux-aarch64/crypto/fipsmodule/sha256-armv8.S index c777ec8209..701e8e19cf 100644 --- a/generated-src/linux-aarch64/crypto/fipsmodule/sha256-armv8.S +++ b/generated-src/linux-aarch64/crypto/fipsmodule/sha256-armv8.S @@ -1073,6 +1073,13 @@ sha256_block_data_order: .align 6 sha256_block_armv8: .Lv8_entry: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#6] // kFlag_sha256_hw +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/linux-aarch64/crypto/fipsmodule/sha512-armv8.S b/generated-src/linux-aarch64/crypto/fipsmodule/sha512-armv8.S index a3b458a24d..6ce23c8cb9 100644 --- a/generated-src/linux-aarch64/crypto/fipsmodule/sha512-armv8.S +++ b/generated-src/linux-aarch64/crypto/fipsmodule/sha512-armv8.S @@ -1097,6 +1097,13 @@ sha512_block_data_order: .align 6 sha512_block_armv8: .Lv8_entry: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#2] // kFlag_sha512_hw +#endif stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S b/generated-src/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S index c02be40fff..16ff2edcaa 100644 --- a/generated-src/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S +++ b/generated-src/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S @@ -217,6 +217,13 @@ _vpaes_encrypt_core: .type vpaes_encrypt,%function .align 4 vpaes_encrypt: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#4] // kFlag_vpaes_encrypt +#endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1020,6 +1027,13 @@ _vpaes_schedule_mangle: .type vpaes_set_encrypt_key,%function .align 4 vpaes_set_encrypt_key: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#5] // kFlag_vpaes_set_encrypt_key +#endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/win-aarch64/crypto/fipsmodule/aesv8-armx.S b/generated-src/win-aarch64/crypto/fipsmodule/aesv8-armx.S index 62993f91e0..67a6c057d2 100644 --- a/generated-src/win-aarch64/crypto/fipsmodule/aesv8-armx.S +++ b/generated-src/win-aarch64/crypto/fipsmodule/aesv8-armx.S @@ -35,6 +35,13 @@ Lrcon: .align 5 aes_hw_set_encrypt_key: Lenc_key: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#3] // kFlag_aes_hw_set_encrypt_key +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! @@ -251,6 +258,13 @@ Ldec_key_abort: .endef .align 5 aes_hw_encrypt: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#1] // kFlag_aes_hw_encrypt +#endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 @@ -284,6 +298,13 @@ Loop_enc: .endef .align 5 aes_hw_decrypt: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#1] // kFlag_aes_hw_encrypt +#endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 @@ -612,6 +633,13 @@ Lcbc_abort: .endef .align 5 aes_hw_ctr32_encrypt_blocks: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6] // kFlag_aes_hw_ctr32_encrypt_blocks +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! diff --git a/generated-src/win-aarch64/crypto/fipsmodule/sha256-armv8.S b/generated-src/win-aarch64/crypto/fipsmodule/sha256-armv8.S index 15970afc29..145e2e1a3d 100644 --- a/generated-src/win-aarch64/crypto/fipsmodule/sha256-armv8.S +++ b/generated-src/win-aarch64/crypto/fipsmodule/sha256-armv8.S @@ -1077,6 +1077,13 @@ LK256: .align 6 sha256_block_armv8: Lv8_entry: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#6] // kFlag_sha256_hw +#endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/win-aarch64/crypto/fipsmodule/sha512-armv8.S b/generated-src/win-aarch64/crypto/fipsmodule/sha512-armv8.S index b2b5a7e7b4..5597227a96 100644 --- a/generated-src/win-aarch64/crypto/fipsmodule/sha512-armv8.S +++ b/generated-src/win-aarch64/crypto/fipsmodule/sha512-armv8.S @@ -1101,6 +1101,13 @@ LK512: .align 6 sha512_block_armv8: Lv8_entry: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#2] // kFlag_sha512_hw +#endif stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/win-aarch64/crypto/fipsmodule/vpaes-armv8.S b/generated-src/win-aarch64/crypto/fipsmodule/vpaes-armv8.S index 522daa68be..41e5837217 100644 --- a/generated-src/win-aarch64/crypto/fipsmodule/vpaes-armv8.S +++ b/generated-src/win-aarch64/crypto/fipsmodule/vpaes-armv8.S @@ -223,6 +223,13 @@ Lenc_entry: .endef .align 4 vpaes_encrypt: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#4] // kFlag_vpaes_encrypt +#endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1050,6 +1057,13 @@ Lschedule_mangle_both: .endef .align 4 vpaes_set_encrypt_key: +#ifdef BORINGSSL_DISPATCH_TEST + + adrp x6,BORINGSSL_function_hit + add x6, x6, :lo12:BORINGSSL_function_hit + mov w7, #1 + strb w7, [x6,#5] // kFlag_vpaes_set_encrypt_key +#endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 From 997345147b5a4dc01894e32e7f99a30e2775cb4f Mon Sep 17 00:00:00 2001 From: Nevine Ebeid <66388554+nebeid@users.noreply.github.com> Date: Thu, 27 Jul 2023 22:38:49 -0400 Subject: [PATCH 2/4] Make dispatch tests use corruptible registers on aarch64. (#1118) Using registers x6 and x7 is risky because they can be used for input parameters. Switching the dispatch tests to use x9 and x10 which are corruptible regitsters. --- crypto/fipsmodule/aes/asm/aesv8-armx.pl | 24 +++++++------- crypto/fipsmodule/aes/asm/vpaes-armv8.pl | 16 +++++----- crypto/fipsmodule/sha/asm/sha512-armv8.pl | 16 +++++----- .../crypto/fipsmodule/aesv8-armx.S | 32 +++++++++---------- .../crypto/fipsmodule/sha256-armv8.S | 8 ++--- .../crypto/fipsmodule/sha512-armv8.S | 8 ++--- .../crypto/fipsmodule/vpaes-armv8.S | 16 +++++----- .../crypto/fipsmodule/aesv8-armx.S | 32 +++++++++---------- .../crypto/fipsmodule/sha256-armv8.S | 8 ++--- .../crypto/fipsmodule/sha512-armv8.S | 8 ++--- .../crypto/fipsmodule/vpaes-armv8.S | 16 +++++----- .../crypto/fipsmodule/aesv8-armx.S | 32 +++++++++---------- .../crypto/fipsmodule/sha256-armv8.S | 8 ++--- .../crypto/fipsmodule/sha512-armv8.S | 8 ++--- .../crypto/fipsmodule/vpaes-armv8.S | 16 +++++----- 15 files changed, 124 insertions(+), 124 deletions(-) diff --git a/crypto/fipsmodule/aes/asm/aesv8-armx.pl b/crypto/fipsmodule/aes/asm/aesv8-armx.pl index 76f4698415..4f2152c0c1 100644 --- a/crypto/fipsmodule/aes/asm/aesv8-armx.pl +++ b/crypto/fipsmodule/aes/asm/aesv8-armx.pl @@ -98,10 +98,10 @@ $code.=<<___ if ($flavour =~ /64/); #ifdef BORINGSSL_DISPATCH_TEST .extern BORINGSSL_function_hit - adrp x6,:pg_hi21:BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#3] // kFlag_aes_hw_set_encrypt_key + adrp x9,:pg_hi21:BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#3] // kFlag_aes_hw_set_encrypt_key #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET @@ -351,10 +351,10 @@ () $code.=<<___ if ($flavour =~ /64/); #ifdef BORINGSSL_DISPATCH_TEST .extern BORINGSSL_function_hit - adrp x6,:pg_hi21:BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#1] // kFlag_aes_hw_encrypt + adrp x9,:pg_hi21:BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#1] // kFlag_aes_hw_encrypt #endif ___ $code.=<<___; @@ -739,10 +739,10 @@ () $code.=<<___ if ($flavour =~ /64/); #ifdef BORINGSSL_DISPATCH_TEST .extern BORINGSSL_function_hit - adrp x6,:pg_hi21:BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6] // kFlag_aes_hw_ctr32_encrypt_blocks + adrp x9,:pg_hi21:BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9] // kFlag_aes_hw_ctr32_encrypt_blocks #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET diff --git a/crypto/fipsmodule/aes/asm/vpaes-armv8.pl b/crypto/fipsmodule/aes/asm/vpaes-armv8.pl index 413633a29c..ea725f83af 100755 --- a/crypto/fipsmodule/aes/asm/vpaes-armv8.pl +++ b/crypto/fipsmodule/aes/asm/vpaes-armv8.pl @@ -263,10 +263,10 @@ vpaes_encrypt: #ifdef BORINGSSL_DISPATCH_TEST .extern BORINGSSL_function_hit - adrp x6,:pg_hi21:BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#4] // kFlag_vpaes_encrypt + adrp x9,:pg_hi21:BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#4] // kFlag_vpaes_encrypt #endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! @@ -1078,10 +1078,10 @@ vpaes_set_encrypt_key: #ifdef BORINGSSL_DISPATCH_TEST .extern BORINGSSL_function_hit - adrp x6,:pg_hi21:BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#5] // kFlag_vpaes_set_encrypt_key + adrp x9,:pg_hi21:BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#5] // kFlag_vpaes_set_encrypt_key #endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! diff --git a/crypto/fipsmodule/sha/asm/sha512-armv8.pl b/crypto/fipsmodule/sha/asm/sha512-armv8.pl index 100bcfe549..c96e4c4cdd 100644 --- a/crypto/fipsmodule/sha/asm/sha512-armv8.pl +++ b/crypto/fipsmodule/sha/asm/sha512-armv8.pl @@ -361,10 +361,10 @@ sub BODY_00_xx { .Lv8_entry: #ifdef BORINGSSL_DISPATCH_TEST .extern BORINGSSL_function_hit - adrp x6,:pg_hi21:BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#6] // kFlag_sha256_hw + adrp x9,:pg_hi21:BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#6] // kFlag_sha256_hw #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! @@ -454,10 +454,10 @@ sub BODY_00_xx { .Lv8_entry: #ifdef BORINGSSL_DISPATCH_TEST .extern BORINGSSL_function_hit - adrp x6,:pg_hi21:BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#2] // kFlag_sha512_hw + adrp x9,:pg_hi21:BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#2] // kFlag_sha512_hw #endif stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/ios-aarch64/crypto/fipsmodule/aesv8-armx.S b/generated-src/ios-aarch64/crypto/fipsmodule/aesv8-armx.S index 9ad7ff41f5..fba86f059d 100644 --- a/generated-src/ios-aarch64/crypto/fipsmodule/aesv8-armx.S +++ b/generated-src/ios-aarch64/crypto/fipsmodule/aesv8-armx.S @@ -34,10 +34,10 @@ _aes_hw_set_encrypt_key: Lenc_key: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,_BORINGSSL_function_hit@PAGE - add x6, x6, _BORINGSSL_function_hit@PAGEOFF - mov w7, #1 - strb w7, [x6,#3] // kFlag_aes_hw_set_encrypt_key + adrp x9,_BORINGSSL_function_hit@PAGE + add x9, x9, _BORINGSSL_function_hit@PAGEOFF + mov w10, #1 + strb w10, [x9,#3] // kFlag_aes_hw_set_encrypt_key #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET @@ -253,10 +253,10 @@ Ldec_key_abort: _aes_hw_encrypt: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,_BORINGSSL_function_hit@PAGE - add x6, x6, _BORINGSSL_function_hit@PAGEOFF - mov w7, #1 - strb w7, [x6,#1] // kFlag_aes_hw_encrypt + adrp x9,_BORINGSSL_function_hit@PAGE + add x9, x9, _BORINGSSL_function_hit@PAGEOFF + mov w10, #1 + strb w10, [x9,#1] // kFlag_aes_hw_encrypt #endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] @@ -291,10 +291,10 @@ Loop_enc: _aes_hw_decrypt: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,_BORINGSSL_function_hit@PAGE - add x6, x6, _BORINGSSL_function_hit@PAGEOFF - mov w7, #1 - strb w7, [x6,#1] // kFlag_aes_hw_encrypt + adrp x9,_BORINGSSL_function_hit@PAGE + add x9, x9, _BORINGSSL_function_hit@PAGEOFF + mov w10, #1 + strb w10, [x9,#1] // kFlag_aes_hw_encrypt #endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] @@ -622,10 +622,10 @@ Lcbc_abort: _aes_hw_ctr32_encrypt_blocks: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,_BORINGSSL_function_hit@PAGE - add x6, x6, _BORINGSSL_function_hit@PAGEOFF - mov w7, #1 - strb w7, [x6] // kFlag_aes_hw_ctr32_encrypt_blocks + adrp x9,_BORINGSSL_function_hit@PAGE + add x9, x9, _BORINGSSL_function_hit@PAGEOFF + mov w10, #1 + strb w10, [x9] // kFlag_aes_hw_ctr32_encrypt_blocks #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET diff --git a/generated-src/ios-aarch64/crypto/fipsmodule/sha256-armv8.S b/generated-src/ios-aarch64/crypto/fipsmodule/sha256-armv8.S index acb4990a0d..e4223885e8 100644 --- a/generated-src/ios-aarch64/crypto/fipsmodule/sha256-armv8.S +++ b/generated-src/ios-aarch64/crypto/fipsmodule/sha256-armv8.S @@ -1074,10 +1074,10 @@ sha256_block_armv8: Lv8_entry: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,_BORINGSSL_function_hit@PAGE - add x6, x6, _BORINGSSL_function_hit@PAGEOFF - mov w7, #1 - strb w7, [x6,#6] // kFlag_sha256_hw + adrp x9,_BORINGSSL_function_hit@PAGE + add x9, x9, _BORINGSSL_function_hit@PAGEOFF + mov w10, #1 + strb w10, [x9,#6] // kFlag_sha256_hw #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! diff --git a/generated-src/ios-aarch64/crypto/fipsmodule/sha512-armv8.S b/generated-src/ios-aarch64/crypto/fipsmodule/sha512-armv8.S index 45621ddfbd..96ef732079 100644 --- a/generated-src/ios-aarch64/crypto/fipsmodule/sha512-armv8.S +++ b/generated-src/ios-aarch64/crypto/fipsmodule/sha512-armv8.S @@ -1098,10 +1098,10 @@ sha512_block_armv8: Lv8_entry: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,_BORINGSSL_function_hit@PAGE - add x6, x6, _BORINGSSL_function_hit@PAGEOFF - mov w7, #1 - strb w7, [x6,#2] // kFlag_sha512_hw + adrp x9,_BORINGSSL_function_hit@PAGE + add x9, x9, _BORINGSSL_function_hit@PAGEOFF + mov w10, #1 + strb w10, [x9,#2] // kFlag_sha512_hw #endif stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/ios-aarch64/crypto/fipsmodule/vpaes-armv8.S b/generated-src/ios-aarch64/crypto/fipsmodule/vpaes-armv8.S index 096dff99d2..9ba3f6133d 100644 --- a/generated-src/ios-aarch64/crypto/fipsmodule/vpaes-armv8.S +++ b/generated-src/ios-aarch64/crypto/fipsmodule/vpaes-armv8.S @@ -218,10 +218,10 @@ Lenc_entry: _vpaes_encrypt: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,_BORINGSSL_function_hit@PAGE - add x6, x6, _BORINGSSL_function_hit@PAGEOFF - mov w7, #1 - strb w7, [x6,#4] // kFlag_vpaes_encrypt + adrp x9,_BORINGSSL_function_hit@PAGE + add x9, x9, _BORINGSSL_function_hit@PAGEOFF + mov w10, #1 + strb w10, [x9,#4] // kFlag_vpaes_encrypt #endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! @@ -1028,10 +1028,10 @@ Lschedule_mangle_both: _vpaes_set_encrypt_key: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,_BORINGSSL_function_hit@PAGE - add x6, x6, _BORINGSSL_function_hit@PAGEOFF - mov w7, #1 - strb w7, [x6,#5] // kFlag_vpaes_set_encrypt_key + adrp x9,_BORINGSSL_function_hit@PAGE + add x9, x9, _BORINGSSL_function_hit@PAGEOFF + mov w10, #1 + strb w10, [x9,#5] // kFlag_vpaes_set_encrypt_key #endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! diff --git a/generated-src/linux-aarch64/crypto/fipsmodule/aesv8-armx.S b/generated-src/linux-aarch64/crypto/fipsmodule/aesv8-armx.S index a8cdd2acab..a9977e2026 100644 --- a/generated-src/linux-aarch64/crypto/fipsmodule/aesv8-armx.S +++ b/generated-src/linux-aarch64/crypto/fipsmodule/aesv8-armx.S @@ -35,10 +35,10 @@ aes_hw_set_encrypt_key: .Lenc_key: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#3] // kFlag_aes_hw_set_encrypt_key + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#3] // kFlag_aes_hw_set_encrypt_key #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET @@ -254,10 +254,10 @@ aes_hw_set_decrypt_key: aes_hw_encrypt: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#1] // kFlag_aes_hw_encrypt + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#1] // kFlag_aes_hw_encrypt #endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] @@ -292,10 +292,10 @@ aes_hw_encrypt: aes_hw_decrypt: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#1] // kFlag_aes_hw_encrypt + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#1] // kFlag_aes_hw_encrypt #endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] @@ -623,10 +623,10 @@ aes_hw_cbc_encrypt: aes_hw_ctr32_encrypt_blocks: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6] // kFlag_aes_hw_ctr32_encrypt_blocks + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9] // kFlag_aes_hw_ctr32_encrypt_blocks #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET diff --git a/generated-src/linux-aarch64/crypto/fipsmodule/sha256-armv8.S b/generated-src/linux-aarch64/crypto/fipsmodule/sha256-armv8.S index 701e8e19cf..d549df843a 100644 --- a/generated-src/linux-aarch64/crypto/fipsmodule/sha256-armv8.S +++ b/generated-src/linux-aarch64/crypto/fipsmodule/sha256-armv8.S @@ -1075,10 +1075,10 @@ sha256_block_armv8: .Lv8_entry: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#6] // kFlag_sha256_hw + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#6] // kFlag_sha256_hw #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! diff --git a/generated-src/linux-aarch64/crypto/fipsmodule/sha512-armv8.S b/generated-src/linux-aarch64/crypto/fipsmodule/sha512-armv8.S index 6ce23c8cb9..bcf4b8d663 100644 --- a/generated-src/linux-aarch64/crypto/fipsmodule/sha512-armv8.S +++ b/generated-src/linux-aarch64/crypto/fipsmodule/sha512-armv8.S @@ -1099,10 +1099,10 @@ sha512_block_armv8: .Lv8_entry: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#2] // kFlag_sha512_hw + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#2] // kFlag_sha512_hw #endif stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S b/generated-src/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S index 16ff2edcaa..28bb4e394e 100644 --- a/generated-src/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S +++ b/generated-src/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S @@ -219,10 +219,10 @@ _vpaes_encrypt_core: vpaes_encrypt: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#4] // kFlag_vpaes_encrypt + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#4] // kFlag_vpaes_encrypt #endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! @@ -1029,10 +1029,10 @@ _vpaes_schedule_mangle: vpaes_set_encrypt_key: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#5] // kFlag_vpaes_set_encrypt_key + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#5] // kFlag_vpaes_set_encrypt_key #endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! diff --git a/generated-src/win-aarch64/crypto/fipsmodule/aesv8-armx.S b/generated-src/win-aarch64/crypto/fipsmodule/aesv8-armx.S index 67a6c057d2..f9743c8c73 100644 --- a/generated-src/win-aarch64/crypto/fipsmodule/aesv8-armx.S +++ b/generated-src/win-aarch64/crypto/fipsmodule/aesv8-armx.S @@ -37,10 +37,10 @@ aes_hw_set_encrypt_key: Lenc_key: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#3] // kFlag_aes_hw_set_encrypt_key + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#3] // kFlag_aes_hw_set_encrypt_key #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET @@ -260,10 +260,10 @@ Ldec_key_abort: aes_hw_encrypt: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#1] // kFlag_aes_hw_encrypt + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#1] // kFlag_aes_hw_encrypt #endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] @@ -300,10 +300,10 @@ Loop_enc: aes_hw_decrypt: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#1] // kFlag_aes_hw_encrypt + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#1] // kFlag_aes_hw_encrypt #endif AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] @@ -635,10 +635,10 @@ Lcbc_abort: aes_hw_ctr32_encrypt_blocks: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6] // kFlag_aes_hw_ctr32_encrypt_blocks + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9] // kFlag_aes_hw_ctr32_encrypt_blocks #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET diff --git a/generated-src/win-aarch64/crypto/fipsmodule/sha256-armv8.S b/generated-src/win-aarch64/crypto/fipsmodule/sha256-armv8.S index 145e2e1a3d..0ed6236ec7 100644 --- a/generated-src/win-aarch64/crypto/fipsmodule/sha256-armv8.S +++ b/generated-src/win-aarch64/crypto/fipsmodule/sha256-armv8.S @@ -1079,10 +1079,10 @@ sha256_block_armv8: Lv8_entry: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#6] // kFlag_sha256_hw + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#6] // kFlag_sha256_hw #endif // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! diff --git a/generated-src/win-aarch64/crypto/fipsmodule/sha512-armv8.S b/generated-src/win-aarch64/crypto/fipsmodule/sha512-armv8.S index 5597227a96..53d003f023 100644 --- a/generated-src/win-aarch64/crypto/fipsmodule/sha512-armv8.S +++ b/generated-src/win-aarch64/crypto/fipsmodule/sha512-armv8.S @@ -1103,10 +1103,10 @@ sha512_block_armv8: Lv8_entry: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#2] // kFlag_sha512_hw + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#2] // kFlag_sha512_hw #endif stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/generated-src/win-aarch64/crypto/fipsmodule/vpaes-armv8.S b/generated-src/win-aarch64/crypto/fipsmodule/vpaes-armv8.S index 41e5837217..6813d7794e 100644 --- a/generated-src/win-aarch64/crypto/fipsmodule/vpaes-armv8.S +++ b/generated-src/win-aarch64/crypto/fipsmodule/vpaes-armv8.S @@ -225,10 +225,10 @@ Lenc_entry: vpaes_encrypt: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#4] // kFlag_vpaes_encrypt + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#4] // kFlag_vpaes_encrypt #endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! @@ -1059,10 +1059,10 @@ Lschedule_mangle_both: vpaes_set_encrypt_key: #ifdef BORINGSSL_DISPATCH_TEST - adrp x6,BORINGSSL_function_hit - add x6, x6, :lo12:BORINGSSL_function_hit - mov w7, #1 - strb w7, [x6,#5] // kFlag_vpaes_set_encrypt_key + adrp x9,BORINGSSL_function_hit + add x9, x9, :lo12:BORINGSSL_function_hit + mov w10, #1 + strb w10, [x9,#5] // kFlag_vpaes_set_encrypt_key #endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! From 55d988e4163c0c2370c82f82c5079f774452c370 Mon Sep 17 00:00:00 2001 From: William Bo Yang Date: Mon, 31 Jul 2023 07:50:50 -0700 Subject: [PATCH 3/4] Support changing OPENSSL_armcap with environment variable on Apple 64-bit ARM systems (#1045) * use handle_cpu_env from cpu_aarch64_linux.c to cpu_aarch64_apple.c * always use OPENSSL_armcap() when detecting hardware capabilities on arm * refactor duplicated into cpu_aarch64.h/c * add ifdefs to cpu_aarch64.h/c * Update comments Co-authored-by: Nevine Ebeid <66388554+nebeid@users.noreply.github.com> * adjust line length of comment * set arm capabilities functions to fillow format of x86 ones * make rest of the arm capability functions in line with x86 ones --------- Co-authored-by: Nevine Ebeid <66388554+nebeid@users.noreply.github.com> Co-authored-by: Bill Yang --- crypto/fipsmodule/bcm.c | 1 + crypto/fipsmodule/cpucap/cpu_aarch64.c | 52 ++++++++++++++++++++ crypto/fipsmodule/cpucap/cpu_aarch64.h | 31 ++++++++++++ crypto/fipsmodule/cpucap/cpu_aarch64_apple.c | 16 ++++++ crypto/fipsmodule/cpucap/cpu_aarch64_linux.c | 35 +------------ 5 files changed, 101 insertions(+), 34 deletions(-) create mode 100644 crypto/fipsmodule/cpucap/cpu_aarch64.c create mode 100644 crypto/fipsmodule/cpucap/cpu_aarch64.h diff --git a/crypto/fipsmodule/bcm.c b/crypto/fipsmodule/bcm.c index 28932bb21d..567890e09e 100644 --- a/crypto/fipsmodule/bcm.c +++ b/crypto/fipsmodule/bcm.c @@ -41,6 +41,7 @@ #include "../internal.h" #include "cpucap/cpucap.c" +#include "cpucap/cpu_aarch64.c" #include "cpucap/cpu_aarch64_apple.c" #include "cpucap/cpu_aarch64_fuchsia.c" #include "cpucap/cpu_aarch64_linux.c" diff --git a/crypto/fipsmodule/cpucap/cpu_aarch64.c b/crypto/fipsmodule/cpucap/cpu_aarch64.c new file mode 100644 index 0000000000..7c085b7e10 --- /dev/null +++ b/crypto/fipsmodule/cpucap/cpu_aarch64.c @@ -0,0 +1,52 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 OR ISC + +#if defined(OPENSSL_AARCH64) && !defined(OPENSSL_STATIC_ARMCAP) + +#include "cpu_aarch64.h" + +void handle_cpu_env(uint32_t *out, const char *in) { + const int invert = in[0] == '~'; + const int or = in[0] == '|'; + const int skip_first_byte = invert || or; + const int hex = in[skip_first_byte] == '0' && in[skip_first_byte+1] == 'x'; + uint32_t armcap = out[0]; + + int sscanf_result; + uint32_t v; + if (hex) { + sscanf_result = sscanf(in + skip_first_byte + 2, "%" PRIx32, &v); + } else { + sscanf_result = sscanf(in + skip_first_byte, "%" PRIu32, &v); + } + + if (!sscanf_result) { + return; + } + + // Detect if the user is trying to use the environment variable to set + // a capability that is _not_ available on the CPU: + // If the runtime capability check (e.g via getauxval() on Linux) + // returned a non-zero hwcap in `armcap` (out) + // and a bit set in the requested `v` is not set in `armcap`, + // abort instead of crashing later. + // The case of invert cannot enable an unexisting capability; + // it can only disable an existing one. + if (!invert && armcap && (~armcap & v)) + { + fprintf(stderr, + "Fatal Error: HW capability found: 0x%02X, but HW capability requested: 0x%02X.\n", + armcap, v); + exit(1); + } + + if (invert) { + out[0] &= ~v; + } else if (or) { + out[0] |= v; + } else { + out[0] = v; + } +} + +#endif // OPENSSL_AARCH64 && !OPENSSL_STATIC_ARMCAP diff --git a/crypto/fipsmodule/cpucap/cpu_aarch64.h b/crypto/fipsmodule/cpucap/cpu_aarch64.h new file mode 100644 index 0000000000..0999d58c53 --- /dev/null +++ b/crypto/fipsmodule/cpucap/cpu_aarch64.h @@ -0,0 +1,31 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 OR ISC + +#ifndef OPENSSL_HEADER_CPUCAP_CPU_AARCH64_H +#define OPENSSL_HEADER_CPUCAP_CPU_AARCH64_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include + +#include +#include +#include + +#if defined(OPENSSL_AARCH64) && !defined(OPENSSL_STATIC_ARMCAP) + +// cpu_aarch64 contains common functions used across multiple cpu_aarch64_* files + +// handle_cpu_env applies the value from |in| to the CPUID values in |out[0]|. +// See the comment in |OPENSSL_cpuid_setup| about this. +void handle_cpu_env(uint32_t *out, const char *in); + +#endif // OPENSSL_AARCH64 && !OPENSSL_STATIC_ARMCAP + +#if defined(__cplusplus) +} +#endif + +#endif // OPENSSL_HEADER_CPUCAP_CPU_AARCH64_H diff --git a/crypto/fipsmodule/cpucap/cpu_aarch64_apple.c b/crypto/fipsmodule/cpucap/cpu_aarch64_apple.c index 92a219a985..b27f447383 100644 --- a/crypto/fipsmodule/cpucap/cpu_aarch64_apple.c +++ b/crypto/fipsmodule/cpucap/cpu_aarch64_apple.c @@ -22,6 +22,7 @@ #include +#include "cpu_aarch64.h" extern uint32_t OPENSSL_armcap_P; @@ -67,6 +68,21 @@ void OPENSSL_cpuid_setup(void) { if (has_hw_feature("hw.optional.armv8_2_sha512")) { OPENSSL_armcap_P |= ARMV8_SHA512; } + + // OPENSSL_armcap is a 32-bit, unsigned value which may start with "0x" to + // indicate a hex value. Prior to the 32-bit value, a '~' or '|' may be given. + // + // If the '~' prefix is present: + // the value is inverted and ANDed with the probed CPUID result + // If the '|' prefix is present: + // the value is ORed with the probed CPUID result + // Otherwise: + // the value is taken as the result of the CPUID + const char *env; + env = getenv("OPENSSL_armcap"); + if (env != NULL) { + handle_cpu_env(&OPENSSL_armcap_P, env); + } } #endif // OPENSSL_AARCH64 && OPENSSL_APPLE && !OPENSSL_STATIC_ARMCAP diff --git a/crypto/fipsmodule/cpucap/cpu_aarch64_linux.c b/crypto/fipsmodule/cpucap/cpu_aarch64_linux.c index 3c19956a18..7611dbc678 100644 --- a/crypto/fipsmodule/cpucap/cpu_aarch64_linux.c +++ b/crypto/fipsmodule/cpucap/cpu_aarch64_linux.c @@ -22,46 +22,13 @@ #ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS #endif -#include - -#include -#include -#include #include +#include "cpu_aarch64.h" extern uint32_t OPENSSL_armcap_P; -// handle_cpu_env applies the value from |in| to the CPUID values in |out[0]| -// and |out[1]|. See the comment in |OPENSSL_cpuid_setup| about this. -static void handle_cpu_env(uint32_t *out, const char *in) { - const int invert = in[0] == '~'; - const int or = in[0] == '|'; - const int skip_first_byte = invert || or; - const int hex = in[skip_first_byte] == '0' && in[skip_first_byte+1] == 'x'; - - int sscanf_result; - uint32_t v; - if (hex) { - sscanf_result = sscanf(in + invert + 2, "%" PRIx32, &v); - } else { - sscanf_result = sscanf(in + invert, "%" PRIu32, &v); - } - - if (!sscanf_result) { - return; - } - - if (invert) { - out[0] &= ~v; - } else if (or) { - out[0] |= v; - } else { - out[0] = v; - } -} - void OPENSSL_cpuid_setup(void) { unsigned long hwcap = getauxval(AT_HWCAP); From b09841895aec429a15ab0cfbd5ae5128103b2097 Mon Sep 17 00:00:00 2001 From: Samuel Chiang Date: Thu, 4 May 2023 12:39:14 -0700 Subject: [PATCH 4/4] fix device farm CI build with boto3 (#990) --- tests/ci/devicefarm_job.py | 2 +- tests/ci/setup.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ci/devicefarm_job.py b/tests/ci/devicefarm_job.py index 24779cd093..c8e0ae1bbe 100644 --- a/tests/ci/devicefarm_job.py +++ b/tests/ci/devicefarm_job.py @@ -20,7 +20,7 @@ "poolArn": DEVICEFARM_DEVICE_POOL, } -client = boto3.client('devicefarm') +client = boto3.client('devicefarm', region_name=AWS_REGION) unique = config['namePrefix'] aws_region = config['awsRegion'] diff --git a/tests/ci/setup.py b/tests/ci/setup.py index 278a9d58bf..966d298a9c 100644 --- a/tests/ci/setup.py +++ b/tests/ci/setup.py @@ -29,10 +29,11 @@ # A formatter for Python code. "yapf==0.30.0", # Introduced by benchmark framework. - "boto3==1.18.11", + "boto3==1.26.126", # Introduced by Android Device Farm CI. "requests", - "arnparse==0.0.2" + "arnparse==0.0.2", + "urllib3==1.25.4" ], python_requires=">=3.6",