diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index e71e4a00c7..336b4e4890 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -785,6 +785,8 @@ WOLFSSL_ALLOW_SERVER_SC_EXT WOLFSSL_ALLOW_TLS_SHA1 WOLFSSL_ALTERNATIVE_DOWNGRADE WOLFSSL_ALT_NAMES_NO_REV +WOLFSSL_ARM32_BUILD +WOLFSSL_ARM32_PRIVILEGE_MODE WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP WOLFSSL_ARM_ARCH_NEON_64BIT WOLFSSL_ASCON_UNROLL diff --git a/configure.ac b/configure.ac index 6116178b05..6ce749b406 100644 --- a/configure.ac +++ b/configure.ac @@ -3917,10 +3917,12 @@ ENABLED_ARMASM_INLINE="no" ENABLED_ARMASM_SHA256_SMALL="no" ENABLED_ARMASM_SHA3="unknown" ENABLED_ARMASM_CRYPTO_SM4="no" +ENABLED_ARMASM_NO_BASE_IMPL="no" +ENABLED_ARMASM_NO_NEON_IMPL="no" # ARM Assembly # Both SHA3 and SHA512 instructions available with ARMV8.2-a AC_ARG_ENABLE([armasm], - [AS_HELP_STRING([--enable-armasm],[Enable wolfSSL ARMv8 ASM support (default: disabled). Set to sha512-crypto or sha3-crypto to use SHA512 and SHA3 instructions with Aarch64 CPU.])], + [AS_HELP_STRING([--enable-armasm],[Enable wolfSSL ARMv8 ASM support (default: disabled). Set to sha512-crypto or sha3-crypto to use SHA512 and SHA3 instructions with Aarch64 CPU. On 32-bit ARM, no-base and no-neon-impl drop the base/NEON run-time fallbacks when the crypto extension is always present.])], [ ENABLED_ARMASM=$enableval ], [ ENABLED_ARMASM=no ] ) @@ -4025,13 +4027,33 @@ then esac ENABLED_ARMASM_AES_BLOCK_INLINE=yes ;; + no-base) + case $host_cpu in + *arm*) + ;; + *) + AC_MSG_ERROR([no-base option only available on 32-bit ARM CPU.]) + break;; + esac + ENABLED_ARMASM_NO_BASE_IMPL=yes + ;; + no-neon-impl) + case $host_cpu in + *arm*) + ;; + *) + AC_MSG_ERROR([no-neon-impl option only available on 32-bit ARM CPU.]) + break;; + esac + ENABLED_ARMASM_NO_NEON_IMPL=yes + ;; *) case $host_cpu in *aarch64*) AC_MSG_ERROR([Invalid choice of ARM asm inclusions (yes, inline, no-crypto, sha512-crypto, sha3-crypto, no-sha512-crypto, no-sha3-crypto, barrier-sb, barrier-detect): $ENABLED_ARMASM.]) break;; *arm*) - AC_MSG_ERROR([Invalid choice of ARM asm inclusions (yes, inline, no-crypto, sha256-small, aes-block-dup): $ENABLED_ARMASM.]) + AC_MSG_ERROR([Invalid choice of ARM asm inclusions (yes, inline, no-crypto, sha256-small, aes-block-dup, no-base, no-neon-impl): $ENABLED_ARMASM.]) break;; esac break;; @@ -4194,6 +4216,17 @@ if test "$ENABLED_ARMASM_NEON" = "no"; then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_ARMASM_NO_NEON" AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_ARMASM_NO_NEON" fi +# 32-bit ARM run-time dispatch: drop a fallback implementation when the crypto +# extension is always present. Gates both the C dispatch and the generated +# assembly, so define for the C and assembler flags. +if test "$ENABLED_ARMASM_NO_BASE_IMPL" = "yes"; then + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_ARMASM_NO_BASE_IMPL" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_ARMASM_NO_BASE_IMPL" +fi +if test "$ENABLED_ARMASM_NO_NEON_IMPL" = "yes"; then + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_ARMASM_NO_NEON_IMPL" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_ARMASM_NO_NEON_IMPL" +fi if test "$ENABLED_ARMASM_INLINE" = "yes"; then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_ARMASM_INLINE" diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 46a89a6ad8..5d0ebee1c5 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -1085,6 +1085,17 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits #endif /* HAVE_AES_DECRYPT */ #elif defined(WOLFSSL_ARMASM) +/* On 32-bit Arm both the base (table) AES and the Armv8 crypto-extension AES + * are compiled into one object by default, and the implementation is chosen at + * run time through aes->use_aes_hw_crypto - mirroring the AArch64 path. The + * base fallback can be dropped with WOLFSSL_ARMASM_NO_BASE_IMPL (crypto + * always present), and WOLFSSL_ARMASM_NO_HW_CRYPTO keeps only the base, both + * of which revert to direct calls with no run-time check. */ +#if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) && \ + !defined(WOLFSSL_ARMASM_NO_BASE_IMPL) + #define WOLFSSL_ARM32_AES_DISPATCH +#endif + #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER; @@ -1098,6 +1109,46 @@ static void Check_CPU_support_HwCrypto(Aes* aes) aes->use_sha3_hw_crypto = IS_AARCH64_SHA3(cpuid_flags); #endif } +#elif defined(WOLFSSL_ARM32_AES_DISPATCH) +static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER; + +/* Record on the Aes object whether this CPU implements the Armv8 AES and PMULL + * crypto-extension instructions, so the per-operation code can select the + * crypto or the base assembly at run time. Called from key setup. + * + * @param [in, out] aes AES object whose use_aes_hw_crypto / + * use_pmull_hw_crypto flags are set. */ +static void Check_CPU_support_HwCrypto(Aes* aes) +{ + if (cpuid_flags == WC_CPUID_INITIALIZER) + cpuid_get_flags_ex(&cpuid_flags); +#ifdef WOLFSSL_AESGCM_STREAM + /* Streaming AES-GCM hashes block-by-block in software over gcm.H in its + * standard form and drives the counter through the base AES_ECB_encrypt, + * both of which need the base key schedule. The crypto-extension AES uses + * a different key schedule and stores H in PMULL-reflected form, so the two + * cannot share one Aes object. When streaming is configured, keep the + * whole cipher on the base implementation. */ + aes->use_aes_hw_crypto = 0; +#ifdef HAVE_AESGCM + aes->use_pmull_hw_crypto = 0; +#endif +#else +#ifdef HAVE_AESGCM + aes->use_pmull_hw_crypto = IS_ARM32_PMULL(cpuid_flags); + /* The crypto and base AES key schedules are incompatible. When PMULL is + * absent, AES-GCM (and AES-GCM-SIV) fall back to the base (software) path, + * which drives its AES through the base AES_ECB_encrypt and so needs the + * base key schedule. Only take the crypto AES path when PMULL is present + * too, so the whole cipher stays consistent. (A CPU implementing AES but + * not PMULL is rare - the crypto extension provides them together.) */ + aes->use_aes_hw_crypto = IS_ARM32_AES(cpuid_flags) && + aes->use_pmull_hw_crypto; +#else + aes->use_aes_hw_crypto = IS_ARM32_AES(cpuid_flags); +#endif +#endif /* WOLFSSL_AESGCM_STREAM */ +} #endif /* __aarch64__ && !WOLFSSL_ARMASM_NO_HW_CRYPTO */ #if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AESCCM) || \ @@ -1107,7 +1158,15 @@ static WARN_UNUSED_RESULT int wc_AesEncrypt(Aes* aes, const byte* inBlock, { #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #if !defined(__aarch64__) +#ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto) { + AES_encrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, + (int)aes->rounds); + } + else +#else AES_encrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, (int)aes->rounds); +#endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto) { AES_encrypt_AARCH64(inBlock, outBlock, (byte*)aes->key, @@ -1122,7 +1181,8 @@ static WARN_UNUSED_RESULT int wc_AesEncrypt(Aes* aes, const byte* inBlock, AES_ECB_encrypt_NEON(inBlock, outBlock, WC_AES_BLOCK_SIZE, (const unsigned char*)aes->key, aes->rounds); } -#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) { AES_ECB_encrypt(inBlock, outBlock, WC_AES_BLOCK_SIZE, (byte*)aes->key, (int)aes->rounds); @@ -1139,7 +1199,15 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt(Aes* aes, const byte* inBlock, { #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #if !defined(__aarch64__) +#ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto) { + AES_decrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, + (int)aes->rounds); + } + else +#else AES_decrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, (int)aes->rounds); +#endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto) { AES_decrypt_AARCH64(inBlock, outBlock, (byte*)aes->key, @@ -1154,7 +1222,8 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt(Aes* aes, const byte* inBlock, AES_ECB_decrypt_NEON(inBlock, outBlock, WC_AES_BLOCK_SIZE, (byte*)aes->key, (int)aes->rounds); } -#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) { AES_ECB_decrypt(inBlock, outBlock, WC_AES_BLOCK_SIZE, (byte*)aes->key, (int)aes->rounds); @@ -3520,7 +3589,15 @@ WC_ALL_ARGS_NOT_NULL static WARN_UNUSED_RESULT int wc_AesEncrypt( #elif defined(WOLFSSL_ARMASM) #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #if !defined(__aarch64__) +#ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto) { + AES_encrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, + (int)aes->rounds); + } + else +#else AES_encrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, (int)aes->rounds); +#endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto) { AES_encrypt_AARCH64(inBlock, outBlock, (byte*)aes->key, @@ -3535,7 +3612,8 @@ WC_ALL_ARGS_NOT_NULL static WARN_UNUSED_RESULT int wc_AesEncrypt( AES_ECB_encrypt_NEON(inBlock, outBlock, WC_AES_BLOCK_SIZE, (const unsigned char*)aes->key, aes->rounds); } -#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) { AES_ECB_encrypt(inBlock, outBlock, WC_AES_BLOCK_SIZE, (const unsigned char*)aes->key, aes->rounds); @@ -4350,7 +4428,15 @@ WC_ALL_ARGS_NOT_NULL static WARN_UNUSED_RESULT int wc_AesDecrypt( #elif defined(WOLFSSL_ARMASM) #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #if !defined(__aarch64__) +#ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto) { + AES_decrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, + (int)aes->rounds); + } + else +#else AES_decrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, (int)aes->rounds); +#endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto) { AES_decrypt_AARCH64(inBlock, outBlock, (byte*)aes->key, @@ -4365,7 +4451,8 @@ WC_ALL_ARGS_NOT_NULL static WARN_UNUSED_RESULT int wc_AesDecrypt( AES_ECB_decrypt_NEON(inBlock, outBlock, WC_AES_BLOCK_SIZE, (const unsigned char*)aes->key, aes->rounds); } -#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) { AES_ECB_decrypt(inBlock, outBlock, WC_AES_BLOCK_SIZE, (const unsigned char*)aes->key, aes->rounds); @@ -4883,17 +4970,28 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt(Aes* aes, const byte* inBlock, aes->rounds = (keylen/4) + 6; #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO - AES_set_key_AARCH32(userKey, keylen, (byte*)aes->key, dir); +#ifdef WOLFSSL_ARM32_AES_DISPATCH + Check_CPU_support_HwCrypto(aes); + if (aes->use_aes_hw_crypto) { + AES_set_key_AARCH32(userKey, keylen, (byte*)aes->key, dir); + } + else #else - AES_set_encrypt_key(userKey, keylen * 8, (byte*)aes->key); + AES_set_key_AARCH32(userKey, keylen, (byte*)aes->key, dir); +#endif /* WOLFSSL_ARM32_AES_DISPATCH */ +#endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */ +#if defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || defined(WOLFSSL_ARM32_AES_DISPATCH) + { + AES_set_encrypt_key(userKey, keylen * 8, (byte*)aes->key); - #ifdef HAVE_AES_DECRYPT - if (dir == AES_DECRYPTION) { - AES_invert_key((byte*)aes->key, aes->rounds); + #ifdef HAVE_AES_DECRYPT + if (dir == AES_DECRYPTION) { + AES_invert_key((byte*)aes->key, aes->rounds); + } + #else + (void)dir; + #endif } - #else - (void)dir; - #endif #endif return wc_AesSetIV(aes, iv); } @@ -5752,7 +5850,15 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #elif defined(WOLFSSL_ARMASM) #if !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) #ifndef __aarch64__ + #ifdef WOLFSSL_ARM32_AES_DISPATCH + Check_CPU_support_HwCrypto(aes); + if (aes->use_aes_hw_crypto) { + AES_set_key_AARCH32(userKey, keylen, (byte*)aes->key, dir); + } + else + #else AES_set_key_AARCH32(userKey, keylen, (byte*)aes->key, dir); + #endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else Check_CPU_support_HwCrypto(aes); if (aes->use_aes_hw_crypto) { @@ -5773,7 +5879,8 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) (void)dir; #endif } - #elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + #elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) { AES_set_encrypt_key(userKey, keylen * 8, (byte*)aes->key); #ifdef HAVE_AES_DECRYPT @@ -7064,8 +7171,16 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) #if defined(WOLFSSL_ARMASM) #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #if !defined(__aarch64__) + #ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto) { + AES_CBC_encrypt_AARCH32(in, out, sz, (byte*)aes->reg, + (byte*)aes->key, (int)aes->rounds); + } + else + #else AES_CBC_encrypt_AARCH32(in, out, sz, (byte*)aes->reg, (byte*)aes->key, (int)aes->rounds); + #endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto) { AES_CBC_encrypt_AARCH64(in, out, sz, (byte*)aes->reg, @@ -7080,7 +7195,8 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) AES_CBC_encrypt_NEON(in, out, sz, (const unsigned char*)aes->key, aes->rounds, (unsigned char*)aes->reg); } - #elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + #elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) { AES_CBC_encrypt(in, out, sz, (const unsigned char*)aes->key, aes->rounds, (unsigned char*)aes->reg); @@ -7300,8 +7416,16 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) #if defined(WOLFSSL_ARMASM) #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #if !defined(__aarch64__) + #ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto) { + AES_CBC_decrypt_AARCH32(in, out, sz, (byte*)aes->reg, + (byte*)aes->key, (int)aes->rounds); + } + else + #else AES_CBC_decrypt_AARCH32(in, out, sz, (byte*)aes->reg, (byte*)aes->key, (int)aes->rounds); + #endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto) { AES_CBC_decrypt_AARCH64(in, out, sz, (byte*)aes->reg, @@ -7322,14 +7446,16 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) else #endif #endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO */ - #if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + #if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP { AES_CBC_decrypt(in, out, sz, (const unsigned char*)aes->key, aes->rounds, (unsigned char*)aes->reg); } #endif - #endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO */ + #endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO || + * WOLFSSL_ARM32_AES_DISPATCH */ return 0; #elif (defined(WOLFSSL_PPC64_ASM) || defined(WOLFSSL_PPC32_ASM)) AES_CBC_decrypt(in, out, sz, (const unsigned char*)aes->key, @@ -7768,8 +7894,17 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) #if defined(WOLFSSL_ARMASM) #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #ifndef __aarch64__ + #ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto) { + AES_CTR_encrypt_AARCH32(in, out, sz, (byte*)aes->reg, + (byte*)aes->key, (byte*)aes->tmp, &aes->left, aes->rounds); + return 0; + } + else + #else AES_CTR_encrypt_AARCH32(in, out, sz, (byte*)aes->reg, (byte*)aes->key, (byte*)aes->tmp, &aes->left, aes->rounds); + #endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto) { AES_CTR_encrypt_AARCH64(in, out, sz, (byte*)aes->reg, @@ -7779,7 +7914,8 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) else #endif /* !__aarch64__ */ #endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */ - #if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + #if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) { word32 numBlocks; byte* tmp = (byte*)aes->tmp + WC_AES_BLOCK_SIZE - aes->left; @@ -8143,7 +8279,7 @@ static WC_INLINE void IncrementGcmCounter(byte* inOutCtr) #endif /* !FREESCALE_LTC_AES_GCM */ #if !defined(WOLFSSL_ARMASM) || defined(__aarch64__) || \ - defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || defined(WOLFSSL_ARM32_AES_DISPATCH) #if defined(GCM_SMALL) || defined(GCM_TABLE) || defined(GCM_TABLE_4BIT) static WC_INLINE void FlattenSzInBits(byte* buf, word32 sz) @@ -8279,7 +8415,13 @@ void GenerateM0(Gcm* gcm) } #endif -#if defined(WOLFSSL_ARMASM) && defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +/* The 32-bit base assembly GHASH (GCM_gmult_len) consumes the M0 table with + * byte-reversed words, so apply that whenever it is compiled in: a no-crypto + * build, or a crypto build that keeps the base fallback for run-time selection + * (WOLFSSL_ARM32_AES_DISPATCH). On AArch64 only the no-crypto build uses the + * M0-table GHASH (the crypto/NEON path hashes H directly). */ +#if defined(WOLFSSL_ARMASM) && (defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH)) for (i = 0; i < 32; i++) { #if !defined(__aarch64__) word32* m32 = (word32*)gcm->M0[i]; @@ -8401,7 +8543,15 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) if (ret == 0) { #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #if !defined(__aarch64__) + #ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { + AES_GCM_set_key_AARCH32(iv, (byte*)aes->key, aes->gcm.H, + aes->rounds); + } + else + #else AES_GCM_set_key_AARCH32(iv, (byte*)aes->key, aes->gcm.H, aes->rounds); + #endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { AES_GCM_set_key_AARCH64(iv, (byte*)aes->key, aes->gcm.H, @@ -8416,7 +8566,8 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) AES_ECB_encrypt_NEON(iv, aes->gcm.H, WC_AES_BLOCK_SIZE, (const unsigned char*)aes->key, aes->rounds); } -#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) { AES_ECB_encrypt(iv, aes->gcm.H, WC_AES_BLOCK_SIZE, (const unsigned char*)aes->key, aes->rounds); @@ -8603,7 +8754,7 @@ void AES_GCM_decrypt_vaes(const unsigned char *in, unsigned char *out, #endif /* WOLFSSL_AESNI */ #if !defined(WOLFSSL_ARMASM) || defined(__aarch64__) || \ - defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || defined(WOLFSSL_ARM32_AES_DISPATCH) #if defined(WOLFSSL_RISCV_SCALAR_CRYPTO_ASM) && defined(HAVE_AESGCM) && \ !defined(WOLFSSL_RISCV_VECTOR_CRYPTO_ASM) /* GHASH using the RISC-V scalar carryless-multiply (Zbc) helper. Vector crypto @@ -9155,7 +9306,8 @@ void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c, #elif defined(GCM_TABLE_4BIT) /* ARM assembly */ #if defined(WOLFSSL_ARMASM) && (defined(__aarch64__) || \ - defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) + defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH)) #if !defined(WOLFSSL_ARMASM_NO_NEON) && defined(__aarch64__) #define GCM_GMULT_LEN(gcm, x, a, len) \ GCM_gmult_len_NEON(x, (const byte*)((gcm)->H), a, len) @@ -10797,6 +10949,7 @@ WARN_UNUSED_RESULT int AES_GCM_encrypt_C( return ret; } #elif (defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) || \ (defined(WOLFSSL_PPC64_ASM) || defined(WOLFSSL_PPC32_ASM)) static int AES_GCM_encrypt_ASM(Aes* aes, byte* out, const byte* in, word32 sz, const byte* iv, word32 ivSz, byte* authTag, word32 authTagSz, @@ -11067,10 +11220,20 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz, #if defined(WOLFSSL_ARMASM) #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #if !defined(__aarch64__) + #ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { + AES_GCM_encrypt_AARCH32(in, out, sz, iv, ivSz, authTag, authTagSz, + authIn, authInSz, (byte*)aes->key, aes->gcm.H, (byte*)aes->tmp, + (byte*)aes->reg, aes->rounds); + ret = 0; + } + else + #else AES_GCM_encrypt_AARCH32(in, out, sz, iv, ivSz, authTag, authTagSz, authIn, authInSz, (byte*)aes->key, aes->gcm.H, (byte*)aes->tmp, (byte*)aes->reg, aes->rounds); ret = 0; + #endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 @@ -11091,12 +11254,14 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz, else #endif /* !__aarch64__ */ #endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */ -#if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +#if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) { ret = AES_GCM_encrypt_ASM(aes, out, in, sz, iv, ivSz, authTag, authTagSz, authIn, authInSz); } -#endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO */ +#endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO || + * WOLFSSL_ARM32_AES_DISPATCH */ #elif (defined(WOLFSSL_PPC64_ASM) || defined(WOLFSSL_PPC32_ASM)) ret = AES_GCM_encrypt_ASM(aes, out, in, sz, iv, ivSz, authTag, authTagSz, authIn, authInSz); @@ -11628,6 +11793,7 @@ int WARN_UNUSED_RESULT AES_GCM_decrypt_C( return ret; } #elif (defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) || \ (defined(WOLFSSL_PPC64_ASM) || defined(WOLFSSL_PPC32_ASM)) static int AES_GCM_decrypt_ASM(Aes* aes, byte* out, const byte* in, word32 sz, const byte* iv, word32 ivSz, const byte* authTag, @@ -11868,6 +12034,9 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz, #if defined(WOLFSSL_ARMASM) #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #ifndef __aarch64__ + #ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) + #endif { #ifdef OPENSSL_EXTRA word32 reg[WC_AES_BLOCK_SIZE / sizeof(word32)]; @@ -11880,6 +12049,9 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz, XMEMCPY(aes->reg, reg, sizeof(reg)); #endif } + #ifdef WOLFSSL_ARM32_AES_DISPATCH + else + #endif #else if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 @@ -11899,12 +12071,14 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz, else #endif /* !__aarch64__ */ #endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */ -#if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +#if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) { ret = AES_GCM_decrypt_ASM(aes, out, in, sz, iv, ivSz, authTag, authTagSz, authIn, authInSz); } -#endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO */ +#endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO || + * WOLFSSL_ARM32_AES_DISPATCH */ #elif (defined(WOLFSSL_PPC64_ASM) || defined(WOLFSSL_PPC32_ASM)) { ret = AES_GCM_decrypt_ASM(aes, out, in, sz, iv, ivSz, authTag, @@ -15803,7 +15977,16 @@ static WARN_UNUSED_RESULT int _AesEcbEncrypt( #if defined(WOLFSSL_RISCV_ASM) AES_encrypt_blocks_RISCV64(in, out, sz, (byte*)aes->key, (int)aes->rounds); #elif !defined(__aarch64__) && defined(WOLFSSL_ARMASM) -#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO +#ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto) { + AES_encrypt_blocks_AARCH32(in, out, sz, (byte*)aes->key, + (int)aes->rounds); + } + else { + AES_ECB_encrypt(in, out, sz, (const unsigned char*)aes->key, + aes->rounds); + } +#elif !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) AES_encrypt_blocks_AARCH32(in, out, sz, (byte*)aes->key, (int)aes->rounds); #else AES_ECB_encrypt(in, out, sz, (const unsigned char*)aes->key, aes->rounds); @@ -15904,7 +16087,16 @@ static WARN_UNUSED_RESULT int _AesEcbDecrypt( #if defined(WOLFSSL_RISCV_ASM) AES_decrypt_blocks_RISCV64(in, out, sz, (byte*)aes->key, (int)aes->rounds); #elif !defined(__aarch64__) && defined(WOLFSSL_ARMASM) -#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO +#ifdef WOLFSSL_ARM32_AES_DISPATCH + if (aes->use_aes_hw_crypto) { + AES_decrypt_blocks_AARCH32(in, out, sz, (byte*)aes->key, + (int)aes->rounds); + } + else { + AES_ECB_decrypt(in, out, sz, (const unsigned char*)aes->key, + aes->rounds); + } +#elif !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) AES_decrypt_blocks_AARCH32(in, out, sz, (byte*)aes->key, (int)aes->rounds); #else AES_ECB_decrypt(in, out, sz, (const unsigned char*)aes->key, aes->rounds); @@ -17312,7 +17504,8 @@ void AES_XTS_decrypt_update_avx512(const unsigned char *in, unsigned char *out, #ifdef HAVE_AES_ECB #if (!defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \ - defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))) || defined(WOLFSSL_AESXTS_STREAM) + defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH)) || defined(WOLFSSL_AESXTS_STREAM) /* helper function for encrypting / decrypting full buffer at once */ static WARN_UNUSED_RESULT int _AesXtsHelper( Aes* aes, byte* out, const byte* in, word32 sz, int dir) @@ -17375,7 +17568,8 @@ static WARN_UNUSED_RESULT int _AesXtsHelper( /* Software AES - XTS Encrypt */ #if (!defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \ - defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))) && !defined(WOLFSSL_PPC64_ASM) + defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH)) && !defined(WOLFSSL_PPC64_ASM) static int AesXtsEncryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, byte *i); @@ -17411,7 +17605,8 @@ static int AesXtsInitTweak_sw(XtsAes* xaes, byte* i) { #endif /* WOLFSSL_AESXTS_STREAM */ #if !defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \ - defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) || defined(WOLFSSL_AESXTS_STREAM) + defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH) || defined(WOLFSSL_AESXTS_STREAM) /* Block-streaming AES-XTS. * * Supply block-aligned input data with successive calls. Final call need not @@ -17564,9 +17759,23 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, ret = 0; #elif !defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + /* The base 32-bit AES assembly has no XTS variant, so the run-time + * fallback is the software XTS (which dispatches per-block via + * wc_AesEncrypt). */ +#ifdef WOLFSSL_ARM32_AES_DISPATCH + if (xaes->aes.use_aes_hw_crypto) { + AES_XTS_encrypt_AARCH32(in, out, sz, i, (byte*)xaes->aes.key, + (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds); + ret = 0; + } + else { + ret = AesXtsEncrypt_sw(xaes, out, in, sz, i); + } +#else AES_XTS_encrypt_AARCH32(in, out, sz, i, (byte*)xaes->aes.key, (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds); ret = 0; +#endif #elif defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); @@ -17902,7 +18111,8 @@ int wc_AesXtsEncryptFinal(XtsAes* xaes, byte* out, const byte* in, word32 sz, /* Software AES - XTS Decrypt */ #if (!defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \ - defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))) && !defined(WOLFSSL_PPC64_ASM) + defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH)) && !defined(WOLFSSL_PPC64_ASM) static int AesXtsDecryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, byte *i); @@ -17923,7 +18133,8 @@ static int AesXtsDecrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, #endif #if (!defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \ - defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))) || defined(WOLFSSL_AESXTS_STREAM) + defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH)) || defined(WOLFSSL_AESXTS_STREAM) /* Block-streaming AES-XTS. * * Same process as encryption but use decrypt key. @@ -18112,9 +18323,23 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, ret = 0; #elif !defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + /* The base 32-bit AES assembly has no XTS variant, so the run-time + * fallback is the software XTS (which dispatches per-block via + * wc_AesDecrypt). */ +#ifdef WOLFSSL_ARM32_AES_DISPATCH + if (xaes->aes.use_aes_hw_crypto) { + AES_XTS_decrypt_AARCH32(in, out, sz, i, (byte*)xaes->aes.key, + (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds); + ret = 0; + } + else { + ret = AesXtsDecrypt_sw(xaes, out, in, sz, i); + } +#else AES_XTS_decrypt_AARCH32(in, out, sz, i, (byte*)xaes->aes.key, (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds); ret = 0; +#endif #elif defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); @@ -19039,23 +19264,28 @@ static WC_INLINE void AesGcmSivByteReverse(byte* out, const byte* in) #elif defined(WOLFSSL_ARMASM) && !defined(__aarch64__) && \ !defined(WOLFSSL_ARMASM_THUMB2) /* AArch32 (32-bit ARM). The generated armv8-32-aes-asm.S provides POLYVAL - * and CTR for the crypto (vmull.p64 / aese) and base (table) variants. - * crypto-vs-base is compile-time (WOLFSSL_ARMASM_NO_HW_CRYPTO) with no - * runtime fallback - the same as the rest of the AArch32 AES. */ + * and CTR for the crypto (vmull.p64 / aese) and base (table) variants. In a + * run-time dispatch build both are compiled in and the selectors below pick + * one per CPU (WOLFSSL_ARM32_AES_DISPATCH); otherwise the choice is fixed + * at compile time by WOLFSSL_ARMASM_NO_HW_CRYPTO - matching the rest of the + * AArch32 AES. */ #define WC_POLYVAL_ASM #define WC_POLYVAL_ASM_AARCH32 #define WC_GCMSIV_CTR_ASM #define WC_GCMSIV_CTR_ASM_AARCH32 - /* The base POLYVAL multiplies through the word64 software table poly->m. It - * is only compiled in the base (no-crypto) build and only when that table - * is built. */ - #if defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) && defined(WORD64_AVAILABLE) && \ + /* The base POLYVAL multiplies through the word64 software table poly->m, + * compiled when the crypto extension can be absent at run time (no-crypto + * build or the run-time dispatch build) and the table is available. */ + #if (defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + defined(WOLFSSL_ARM32_AES_DISPATCH)) && defined(WORD64_AVAILABLE) && \ !defined(GCM_WORD32) && !defined(GCM_SMALL) #define WC_POLYVAL_ASM_AARCH32_BASE #endif #ifdef __cplusplus extern "C" { #endif + /* Crypto and base variants both exist in a dispatch build; the selectors + * below pick one per CPU at run time. */ #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO void AES_GCMSIV_polyval_crypto(unsigned char* s, const unsigned char* h, const unsigned char* data, word32 blocks) @@ -19063,7 +19293,8 @@ static WC_INLINE void AesGcmSivByteReverse(byte* out, const byte* in) void AES_GCMSIV_ctr_crypto(const unsigned char* in, unsigned char* out, unsigned long length, const unsigned char* KS, int nr, unsigned char* ctr) XASM_LINK("AES_GCMSIV_ctr_crypto"); -#else +#endif +#if defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || defined(WOLFSSL_ARM32_AES_DISPATCH) #ifdef WC_POLYVAL_ASM_AARCH32_BASE void AES_GCMSIV_polyval_base(unsigned char* s, const unsigned char* h, const unsigned char* data, word32 blocks) @@ -19450,10 +19681,28 @@ static AesGcmSivPolyvalFn AesGcmSivPolyvalAsm(void) return &AES_GCMSIV_polyval_thumb2; } #elif defined(WC_POLYVAL_ASM_AARCH32) -/* AArch32: crypto (vmull.p64) or base (table) POLYVAL, chosen at compile time. */ +/* AArch32: crypto (vmull.p64) POLYVAL when the CPU implements PMULL, else the + * base (table) variant. In a run-time dispatch build the choice is made per + * CPU; otherwise it is fixed at compile time. Streaming keeps the whole cipher + * on the base implementation (see Check_CPU_support_HwCrypto), so select base + * to match. */ static AesGcmSivPolyvalFn AesGcmSivPolyvalAsm(void) { -#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO +#ifdef WOLFSSL_ARM32_AES_DISPATCH +#ifndef WOLFSSL_AESGCM_STREAM + cpuid_get_flags_ex(&cpuid_flags); + if (IS_ARM32_PMULL(cpuid_flags)) { + return &AES_GCMSIV_polyval_crypto; + } +#endif + /* The base multiply needs the word64 table (poly->m), which is not built + * for GCM_SMALL / GCM_WORD32; fall back to the C multiply there. */ +#ifdef WC_POLYVAL_ASM_AARCH32_BASE + return &AES_GCMSIV_polyval_base; +#else + return NULL; +#endif +#elif !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) return &AES_GCMSIV_polyval_crypto; #elif defined(WC_POLYVAL_ASM_AARCH32_BASE) return &AES_GCMSIV_polyval_base; @@ -19517,10 +19766,22 @@ static AesGcmSivCtrFn AesGcmSivCtrAsm(void) return &AES_GCMSIV_ctr_thumb2; } #elif defined(WC_GCMSIV_CTR_ASM_AARCH32) -/* AArch32: crypto (aese) or base (table) CTR, chosen at compile time. */ +/* AArch32: crypto (aese) CTR when the CPU implements AES, else the base (table) + * variant. The CTR keystream runs through the AES key schedule, so the variant + * must match how the key was expanded (Check_CPU_support_HwCrypto): the crypto + * schedule is taken only when both AES and PMULL are present, and streaming + * forces the base schedule. */ static AesGcmSivCtrFn AesGcmSivCtrAsm(void) { -#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO +#ifdef WOLFSSL_ARM32_AES_DISPATCH +#ifndef WOLFSSL_AESGCM_STREAM + cpuid_get_flags_ex(&cpuid_flags); + if (IS_ARM32_AES(cpuid_flags) && IS_ARM32_PMULL(cpuid_flags)) { + return &AES_GCMSIV_ctr_crypto; + } +#endif + return &AES_GCMSIV_ctr_base; +#elif !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) return &AES_GCMSIV_ctr_crypto; #else return &AES_GCMSIV_ctr_base; diff --git a/wolfcrypt/src/cpuid.c b/wolfcrypt/src/cpuid.c index 854a34ff53..5670cabbee 100644 --- a/wolfcrypt/src/cpuid.c +++ b/wolfcrypt/src/cpuid.c @@ -24,7 +24,8 @@ #include #if defined(HAVE_CPUID) || defined(HAVE_CPUID_INTEL) || \ - defined(HAVE_CPUID_AARCH64) || defined(HAVE_CPUID_PPC64) + defined(HAVE_CPUID_AARCH64) || defined(HAVE_CPUID_ARM32) || \ + defined(HAVE_CPUID_PPC64) static cpuid_flags_atomic_t cpuid_flags = WC_CPUID_ATOMIC_INITIALIZER; #endif @@ -567,6 +568,256 @@ } } #endif +#elif defined(HAVE_CPUID_ARM32) + +/* Run-time feature detection for 32-bit Armv8-A (AArch32). + * + * Only the extensions that gate an available wolfCrypt AArch32 assembly + * implementation are detected: the Armv8 crypto extension (AES, PMULL and + * SHA-256) and Advanced SIMD (NEON). There is no AArch32 crypto-extension + * SHA-512/SHA-3, so those digests are handled by the NEON assembly. */ + +/* Fallback flags used ONLY when no run-time detection is available (the #else + * branch below): assume whatever the code was compiled to target. The real + * detection branches must start from zero instead - a build that can emit the + * crypto instructions (__ARM_FEATURE_CRYPTO) does not imply the CPU running the + * binary implements them, which is the whole point of detecting at run time. */ +#if defined(__ARM_FEATURE_CRYPTO) && !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + #define CPUID_ARM32_COMPILED (CPUID_AES | CPUID_PMULL | CPUID_SHA256) +#else + #define CPUID_ARM32_COMPILED 0 +#endif + +#ifdef WOLFSSL_ARM32_PRIVILEGE_MODE + /* Read the crypto feature fields directly from ID_ISAR5. MRC to CP15 is a + * privileged operation, so this path requires the code to run at PL1/EL1. + * https://developer.arm.com/documentation/ddi0595/2021-12/AArch32-Registers + * /ID-ISAR5--Instruction-Set-Attribute-Register-5 */ + + /* ID_ISAR5.AES [7:4]: 1 => AESE/AESD/AESMC/AESIMC, 2 => adds VMULL.P64. */ + #define CPUID_ARM32_ISAR5_AES(isar5) (((isar5) >> 4) & 0xf) + /* ID_ISAR5.SHA2 [15:12]: 1 => SHA256H/SHA256H2/SHA256SU0/SHA256SU1. */ + #define CPUID_ARM32_ISAR5_SHA2(isar5) (((isar5) >> 12) & 0xf) + + static WC_INLINE void cpuid_set_flags(void) + { + if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) { + cpuid_flags_t new_cpuid_flags = 0, + old_cpuid_flags = WC_CPUID_INITIALIZER; + + #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO + word32 isar5; + word32 aes; + + __asm__ __volatile__ ( + "mrc p15, 0, %[reg], c0, c2, 5\n" + : [reg] "=r" (isar5) + : + : + ); + + aes = CPUID_ARM32_ISAR5_AES(isar5); + if (aes >= 1) + new_cpuid_flags |= CPUID_AES; + if (aes >= 2) + new_cpuid_flags |= CPUID_PMULL; + if (CPUID_ARM32_ISAR5_SHA2(isar5) >= 1) + new_cpuid_flags |= CPUID_SHA256; + #endif + #ifndef WOLFSSL_ARMASM_NO_NEON + /* AArch32 has no ID_ISAR bit for Advanced SIMD; whether NEON is + * present is taken from the architecture the code is built for. */ + #ifdef __ARM_NEON + new_cpuid_flags |= CPUID_ASIMD; + #endif + #endif + + (void)wolfSSL_Atomic_Uint_CompareExchange + (&cpuid_flags, &old_cpuid_flags, new_cpuid_flags); + } + } +#elif defined(__linux__) + /* getauxval() reports NEON in AT_HWCAP and the crypto-extension features in + * AT_HWCAP2 on 32-bit Arm. */ + + #include + #include + + /* Feature bits, defined here in case the kernel headers predate them. */ + #ifndef HWCAP_NEON + #define HWCAP_NEON (1 << 12) + #endif + #ifndef HWCAP2_AES + #define HWCAP2_AES (1 << 0) + #endif + #ifndef HWCAP2_PMULL + #define HWCAP2_PMULL (1 << 1) + #endif + #ifndef HWCAP2_SHA2 + #define HWCAP2_SHA2 (1 << 3) + #endif + + static WC_INLINE void cpuid_set_flags(void) + { + if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) { + cpuid_flags_t new_cpuid_flags = 0, + old_cpuid_flags = WC_CPUID_INITIALIZER; + #ifndef WOLFSSL_ARMASM_NO_NEON + unsigned long hwcaps = getauxval(AT_HWCAP); + #endif + #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO + unsigned long hwcaps2 = getauxval(AT_HWCAP2); + #endif + + #ifndef WOLFSSL_ARMASM_NO_NEON + if (hwcaps & HWCAP_NEON) + new_cpuid_flags |= CPUID_ASIMD; + #endif + #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO + if (hwcaps2 & HWCAP2_AES) + new_cpuid_flags |= CPUID_AES; + if (hwcaps2 & HWCAP2_PMULL) + new_cpuid_flags |= CPUID_PMULL; + if (hwcaps2 & HWCAP2_SHA2) + new_cpuid_flags |= CPUID_SHA256; + #endif + + (void)wolfSSL_Atomic_Uint_CompareExchange + (&cpuid_flags, &old_cpuid_flags, new_cpuid_flags); + } + } +#elif defined(__ANDROID__) || defined(ANDROID) + #include "cpu-features.h" + + static WC_INLINE void cpuid_set_flags(void) + { + if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) { + cpuid_flags_t new_cpuid_flags = 0, + old_cpuid_flags = WC_CPUID_INITIALIZER; + #if !defined(WOLFSSL_ARMASM_NO_NEON) || \ + !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + uint64_t features = android_getCpuFeatures(); + #endif + + #ifndef WOLFSSL_ARMASM_NO_NEON + if (features & ANDROID_CPU_ARM_FEATURE_NEON) + new_cpuid_flags |= CPUID_ASIMD; + #endif + #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO + if (features & ANDROID_CPU_ARM_FEATURE_AES) + new_cpuid_flags |= CPUID_AES; + if (features & ANDROID_CPU_ARM_FEATURE_PMULL) + new_cpuid_flags |= CPUID_PMULL; + if (features & ANDROID_CPU_ARM_FEATURE_SHA2) + new_cpuid_flags |= CPUID_SHA256; + #endif + + (void)wolfSSL_Atomic_Uint_CompareExchange + (&cpuid_flags, &old_cpuid_flags, new_cpuid_flags); + } + } +#elif defined(__FreeBSD__) || defined(__OpenBSD__) + #include + + /* Feature bits, defined here in case the system headers do not provide + * them for the 32-bit Arm target. */ + #ifndef HWCAP_NEON + #define HWCAP_NEON (1 << 12) + #endif + #ifndef HWCAP2_AES + #define HWCAP2_AES (1 << 0) + #endif + #ifndef HWCAP2_PMULL + #define HWCAP2_PMULL (1 << 1) + #endif + #ifndef HWCAP2_SHA2 + #define HWCAP2_SHA2 (1 << 3) + #endif + + static WC_INLINE void cpuid_set_flags(void) + { + if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) { + cpuid_flags_t new_cpuid_flags = 0, + old_cpuid_flags = WC_CPUID_INITIALIZER; + #ifndef WOLFSSL_ARMASM_NO_NEON + unsigned long hwcaps = 0; + #endif + #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO + unsigned long hwcaps2 = 0; + #endif + + #ifndef WOLFSSL_ARMASM_NO_NEON + elf_aux_info(AT_HWCAP, &hwcaps, sizeof(hwcaps)); + if (hwcaps & HWCAP_NEON) + new_cpuid_flags |= CPUID_ASIMD; + #endif + #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO + elf_aux_info(AT_HWCAP2, &hwcaps2, sizeof(hwcaps2)); + if (hwcaps2 & HWCAP2_AES) + new_cpuid_flags |= CPUID_AES; + if (hwcaps2 & HWCAP2_PMULL) + new_cpuid_flags |= CPUID_PMULL; + if (hwcaps2 & HWCAP2_SHA2) + new_cpuid_flags |= CPUID_SHA256; + #endif + + (void)wolfSSL_Atomic_Uint_CompareExchange + (&cpuid_flags, &old_cpuid_flags, new_cpuid_flags); + } + } +#elif defined(_WIN32) + /* Windows on 32-bit Arm. IsProcessorFeaturePresent() exposes NEON and the + * mandatory Armv8 crypto extension (AES / PMULL / SHA-1 / SHA-256, reported + * as a single feature). */ + #include + + #ifndef PF_ARM_NEON_INSTRUCTIONS_AVAILABLE + #define PF_ARM_NEON_INSTRUCTIONS_AVAILABLE 19 + #endif + #ifndef PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE + #define PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE 30 + #endif + + static WC_INLINE void cpuid_set_flags(void) + { + if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) { + cpuid_flags_t new_cpuid_flags = 0, + old_cpuid_flags = WC_CPUID_INITIALIZER; + + #ifndef WOLFSSL_ARMASM_NO_NEON + if (IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE)) + new_cpuid_flags |= CPUID_ASIMD; + #endif + #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO + if (IsProcessorFeaturePresent( + PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE)) { + new_cpuid_flags |= CPUID_AES; + new_cpuid_flags |= CPUID_PMULL; + new_cpuid_flags |= CPUID_SHA256; + } + #endif + + (void)wolfSSL_Atomic_Uint_CompareExchange + (&cpuid_flags, &old_cpuid_flags, new_cpuid_flags); + } + } +#else + /* No run-time detection available - report only what the code was compiled + * to require. */ + static WC_INLINE void cpuid_set_flags(void) + { + if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) { + cpuid_flags_t new_cpuid_flags = CPUID_ARM32_COMPILED, + old_cpuid_flags = WC_CPUID_INITIALIZER; + #if !defined(WOLFSSL_ARMASM_NO_NEON) && defined(__ARM_NEON) + new_cpuid_flags |= CPUID_ASIMD; + #endif + + (void)wolfSSL_Atomic_Uint_CompareExchange + (&cpuid_flags, &old_cpuid_flags, new_cpuid_flags); + } + } +#endif #elif defined(HAVE_CPUID_PPC64) /* PowerPC feature bits as reported through the ELF auxiliary vector diff --git a/wolfcrypt/src/port/arm/armv8-32-aes-asm.S b/wolfcrypt/src/port/arm/armv8-32-aes-asm.S index 447b8923fc..5e4e6d64ff 100644 --- a/wolfcrypt/src/port/arm/armv8-32-aes-asm.S +++ b/wolfcrypt/src/port/arm/armv8-32-aes-asm.S @@ -31,9005 +31,1609 @@ #if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2) #ifndef WOLFSSL_ARMASM_INLINE #ifndef NO_AES -#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO +#if !defined(WOLFSSL_ARMASM_NO_BASE_IMPL) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +#ifdef HAVE_AES_DECRYPT +#ifndef __APPLE__ + .text + .type L_AES_ARM32_td_data, %object + .size L_AES_ARM32_td_data, 1024 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_AES_ARM32_td_data: + .long 0x5051f4a7,0x537e4165,0xc31a17a4,0x963a275e + .long 0xcb3bab6b,0xf11f9d45,0xabacfa58,0x934be303 + .long 0x552030fa,0xf6ad766d,0x9188cc76,0x25f5024c + .long 0xfc4fe5d7,0xd7c52acb,0x80263544,0x8fb562a3 + .long 0x49deb15a,0x6725ba1b,0x9845ea0e,0xe15dfec0 + .long 0x02c32f75,0x12814cf0,0xa38d4697,0xc66bd3f9 + .long 0xe7038f5f,0x9515929c,0xebbf6d7a,0xda955259 + .long 0x2dd4be83,0xd3587421,0x2949e069,0x448ec9c8 + .long 0x6a75c289,0x78f48e79,0x6b99583e,0xdd27b971 + .long 0xb6bee14f,0x17f088ad,0x66c920ac,0xb47dce3a + .long 0x1863df4a,0x82e51a31,0x60975133,0x4562537f + .long 0xe0b16477,0x84bb6bae,0x1cfe81a0,0x94f9082b + .long 0x58704868,0x198f45fd,0x8794de6c,0xb7527bf8 + .long 0x23ab73d3,0xe2724b02,0x57e31f8f,0x2a6655ab + .long 0x07b2eb28,0x032fb5c2,0x9a86c57b,0xa5d33708 + .long 0xf2302887,0xb223bfa5,0xba02036a,0x5ced1682 + .long 0x2b8acf1c,0x92a779b4,0xf0f307f2,0xa14e69e2 + .long 0xcd65daf4,0xd50605be,0x1fd13462,0x8ac4a6fe + .long 0x9d342e53,0xa0a2f355,0x32058ae1,0x75a4f6eb + .long 0x390b83ec,0xaa4060ef,0x065e719f,0x51bd6e10 + .long 0xf93e218a,0x3d96dd06,0xaedd3e05,0x464de6bd + .long 0xb591548d,0x0571c45d,0x6f0406d4,0xff605015 + .long 0x241998fb,0x97d6bde9,0xcc894043,0x7767d99e + .long 0xbdb0e842,0x8807898b,0x38e7195b,0xdb79c8ee + .long 0x47a17c0a,0xe97c420f,0xc9f8841e,0x00000000 + .long 0x83098086,0x48322bed,0xac1e1170,0x4e6c5a72 + .long 0xfbfd0eff,0x560f8538,0x1e3daed5,0x27362d39 + .long 0x640a0fd9,0x21685ca6,0xd19b5b54,0x3a24362e + .long 0xb10c0a67,0x0f9357e7,0xd2b4ee96,0x9e1b9b91 + .long 0x4f80c0c5,0xa261dc20,0x695a774b,0x161c121a + .long 0x0ae293ba,0xe5c0a02a,0x433c22e0,0x1d121b17 + .long 0x0b0e090d,0xadf28bc7,0xb92db6a8,0xc8141ea9 + .long 0x8557f119,0x4caf7507,0xbbee99dd,0xfda37f60 + .long 0x9ff70126,0xbc5c72f5,0xc544663b,0x345bfb7e + .long 0x768b4329,0xdccb23c6,0x68b6edfc,0x63b8e4f1 + .long 0xcad731dc,0x10426385,0x40139722,0x2084c611 + .long 0x7d854a24,0xf8d2bb3d,0x11aef932,0x6dc729a1 + .long 0x4b1d9e2f,0xf3dcb230,0xec0d8652,0xd077c1e3 + .long 0x6c2bb316,0x99a970b9,0xfa119448,0x2247e964 + .long 0xc4a8fc8c,0x1aa0f03f,0xd8567d2c,0xef223390 + .long 0xc787494e,0xc1d938d1,0xfe8ccaa2,0x3698d40b + .long 0xcfa6f581,0x28a57ade,0x26dab78e,0xa43fadbf + .long 0xe42c3a9d,0x0d507892,0x9b6a5fcc,0x62547e46 + .long 0xc2f68d13,0xe890d8b8,0x5e2e39f7,0xf582c3af + .long 0xbe9f5d80,0x7c69d093,0xa96fd52d,0xb3cf2512 + .long 0x3bc8ac99,0xa710187d,0x6ee89c63,0x7bdb3bbb + .long 0x09cd2678,0xf46e5918,0x01ec9ab7,0xa8834f9a + .long 0x65e6956e,0x7eaaffe6,0x0821bccf,0xe6ef15e8 + .long 0xd9bae79b,0xce4a6f36,0xd4ea9f09,0xd629b07c + .long 0xaf31a4b2,0x312a3f23,0x30c6a594,0xc035a266 + .long 0x37744ebc,0xa6fc82ca,0xb0e090d0,0x1533a7d8 + .long 0x4af10498,0xf741ecda,0x0e7fcd50,0x2f1791f6 + .long 0x8d764dd6,0x4d43efb0,0x54ccaa4d,0xdfe49604 + .long 0xe39ed1b5,0x1b4c6a88,0xb8c12c1f,0x7f466551 + .long 0x049d5eea,0x5d018c35,0x73fa8774,0x2efb0b41 + .long 0x5ab3671d,0x5292dbd2,0x33e91056,0x136dd647 + .long 0x8c9ad761,0x7a37a10c,0x8e59f814,0x89eb133c + .long 0xeecea927,0x35b761c9,0xede11ce5,0x3c7a47b1 + .long 0x599cd2df,0x3f55f273,0x791814ce,0xbf73c737 + .long 0xea53f7cd,0x5b5ffdaa,0x14df3d6f,0x867844db + .long 0x81caaff3,0x3eb968c4,0x2c382434,0x5fc2a340 + .long 0x72161dc3,0x0cbce225,0x8b283c49,0x41ff0d95 + .long 0x7139a801,0xde080cb3,0x9cd8b4e4,0x906456c1 + .long 0x617bcb84,0x70d532b6,0x74486c5c,0x42d0b857 +#endif /* HAVE_AES_DECRYPT */ +#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AES_CBC) || \ + defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ + defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) +#ifndef __APPLE__ + .text + .type L_AES_ARM32_te_data, %object + .size L_AES_ARM32_te_data, 1024 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_AES_ARM32_te_data: + .long 0xa5c66363,0x84f87c7c,0x99ee7777,0x8df67b7b + .long 0x0dfff2f2,0xbdd66b6b,0xb1de6f6f,0x5491c5c5 + .long 0x50603030,0x03020101,0xa9ce6767,0x7d562b2b + .long 0x19e7fefe,0x62b5d7d7,0xe64dabab,0x9aec7676 + .long 0x458fcaca,0x9d1f8282,0x4089c9c9,0x87fa7d7d + .long 0x15effafa,0xebb25959,0xc98e4747,0x0bfbf0f0 + .long 0xec41adad,0x67b3d4d4,0xfd5fa2a2,0xea45afaf + .long 0xbf239c9c,0xf753a4a4,0x96e47272,0x5b9bc0c0 + .long 0xc275b7b7,0x1ce1fdfd,0xae3d9393,0x6a4c2626 + .long 0x5a6c3636,0x417e3f3f,0x02f5f7f7,0x4f83cccc + .long 0x5c683434,0xf451a5a5,0x34d1e5e5,0x08f9f1f1 + .long 0x93e27171,0x73abd8d8,0x53623131,0x3f2a1515 + .long 0x0c080404,0x5295c7c7,0x65462323,0x5e9dc3c3 + .long 0x28301818,0xa1379696,0x0f0a0505,0xb52f9a9a + .long 0x090e0707,0x36241212,0x9b1b8080,0x3ddfe2e2 + .long 0x26cdebeb,0x694e2727,0xcd7fb2b2,0x9fea7575 + .long 0x1b120909,0x9e1d8383,0x74582c2c,0x2e341a1a + .long 0x2d361b1b,0xb2dc6e6e,0xeeb45a5a,0xfb5ba0a0 + .long 0xf6a45252,0x4d763b3b,0x61b7d6d6,0xce7db3b3 + .long 0x7b522929,0x3edde3e3,0x715e2f2f,0x97138484 + .long 0xf5a65353,0x68b9d1d1,0x00000000,0x2cc1eded + .long 0x60402020,0x1fe3fcfc,0xc879b1b1,0xedb65b5b + .long 0xbed46a6a,0x468dcbcb,0xd967bebe,0x4b723939 + .long 0xde944a4a,0xd4984c4c,0xe8b05858,0x4a85cfcf + .long 0x6bbbd0d0,0x2ac5efef,0xe54faaaa,0x16edfbfb + .long 0xc5864343,0xd79a4d4d,0x55663333,0x94118585 + .long 0xcf8a4545,0x10e9f9f9,0x06040202,0x81fe7f7f + .long 0xf0a05050,0x44783c3c,0xba259f9f,0xe34ba8a8 + .long 0xf3a25151,0xfe5da3a3,0xc0804040,0x8a058f8f + .long 0xad3f9292,0xbc219d9d,0x48703838,0x04f1f5f5 + .long 0xdf63bcbc,0xc177b6b6,0x75afdada,0x63422121 + .long 0x30201010,0x1ae5ffff,0x0efdf3f3,0x6dbfd2d2 + .long 0x4c81cdcd,0x14180c0c,0x35261313,0x2fc3ecec + .long 0xe1be5f5f,0xa2359797,0xcc884444,0x392e1717 + .long 0x5793c4c4,0xf255a7a7,0x82fc7e7e,0x477a3d3d + .long 0xacc86464,0xe7ba5d5d,0x2b321919,0x95e67373 + .long 0xa0c06060,0x98198181,0xd19e4f4f,0x7fa3dcdc + .long 0x66442222,0x7e542a2a,0xab3b9090,0x830b8888 + .long 0xca8c4646,0x29c7eeee,0xd36bb8b8,0x3c281414 + .long 0x79a7dede,0xe2bc5e5e,0x1d160b0b,0x76addbdb + .long 0x3bdbe0e0,0x56643232,0x4e743a3a,0x1e140a0a + .long 0xdb924949,0x0a0c0606,0x6c482424,0xe4b85c5c + .long 0x5d9fc2c2,0x6ebdd3d3,0xef43acac,0xa6c46262 + .long 0xa8399191,0xa4319595,0x37d3e4e4,0x8bf27979 + .long 0x32d5e7e7,0x438bc8c8,0x596e3737,0xb7da6d6d + .long 0x8c018d8d,0x64b1d5d5,0xd29c4e4e,0xe049a9a9 + .long 0xb4d86c6c,0xfaac5656,0x07f3f4f4,0x25cfeaea + .long 0xafca6565,0x8ef47a7a,0xe947aeae,0x18100808 + .long 0xd56fbaba,0x88f07878,0x6f4a2525,0x725c2e2e + .long 0x24381c1c,0xf157a6a6,0xc773b4b4,0x5197c6c6 + .long 0x23cbe8e8,0x7ca1dddd,0x9ce87474,0x213e1f1f + .long 0xdd964b4b,0xdc61bdbd,0x860d8b8b,0x850f8a8a + .long 0x90e07070,0x427c3e3e,0xc471b5b5,0xaacc6666 + .long 0xd8904848,0x05060303,0x01f7f6f6,0x121c0e0e + .long 0xa3c26161,0x5f6a3535,0xf9ae5757,0xd069b9b9 + .long 0x91178686,0x5899c1c1,0x273a1d1d,0xb9279e9e + .long 0x38d9e1e1,0x13ebf8f8,0xb32b9898,0x33221111 + .long 0xbbd26969,0x70a9d9d9,0x89078e8e,0xa7339494 + .long 0xb62d9b9b,0x223c1e1e,0x92158787,0x20c9e9e9 + .long 0x4987cece,0xffaa5555,0x78502828,0x7aa5dfdf + .long 0x8f038c8c,0xf859a1a1,0x80098989,0x171a0d0d + .long 0xda65bfbf,0x31d7e6e6,0xc6844242,0xb8d06868 + .long 0xc3824141,0xb0299999,0x775a2d2d,0x111e0f0f + .long 0xcb7bb0b0,0xfca85454,0xd66dbbbb,0x3a2c1616 +#endif /* HAVE_AES_DECRYPT || HAVE_AES_CBC || HAVE_AESCCM || HAVE_AESGCM || + * WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */ +#ifdef HAVE_AES_DECRYPT +#ifndef __APPLE__ + .text + .type L_AES_ARM32_td, %object + .size L_AES_ARM32_td, 12 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_AES_ARM32_td: + .long L_AES_ARM32_td_data +#endif /* HAVE_AES_DECRYPT */ +#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AES_CBC) || \ + defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ + defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) +#ifndef __APPLE__ + .text + .type L_AES_ARM32_te, %object + .size L_AES_ARM32_te, 12 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_AES_ARM32_te: + .long L_AES_ARM32_te_data +#endif /* HAVE_AES_DECRYPT || HAVE_AES_CBC || HAVE_AESCCM || HAVE_AESGCM || + * WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */ +#ifdef HAVE_AES_DECRYPT .text .align 4 - .globl AES_set_key_AARCH32 - .type AES_set_key_AARCH32, %function -AES_set_key_AARCH32: + .globl AES_invert_key + .type AES_invert_key, %function +AES_invert_key: push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - cmp r1, #24 - blt L_aes_set_key_arm32_crypto_start_128 - bgt L_aes_set_key_arm32_crypto_start_256 - ldr r4, [r0], #4 - ldr r5, [r0], #4 - ldr r6, [r0], #4 - ldr r7, [r0], #4 - ldr r8, [r0], #4 - ldr r9, [r0], #4 + adr r12, L_AES_ARM32_te + ldr r12, [r12] + adr lr, L_AES_ARM32_td + ldr lr, [lr] + add r10, r0, r1, lsl #4 + mov r11, r1 +L_AES_invert_key_loop: + ldm r0, {r2, r3, r4, r5} + ldm r10, {r6, r7, r8, r9} + stm r10, {r2, r3, r4, r5} + stm r0!, {r6, r7, r8, r9} + subs r11, r11, #2 + sub r10, r10, #16 + bne L_AES_invert_key_loop + sub r0, r0, r1, lsl #3 + add r0, r0, #16 + sub r11, r1, #1 +L_AES_invert_key_mix_loop: + ldm r0, {r2, r3, r4, r5} #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r2, #24 + lsr r6, r6, #24 #else - strd r4, r5, [r2], #8 + uxtb r6, r2 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r6, r2, #0, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r2, #16 + lsr r7, r7, #24 #else - strd r8, r9, [r2], #8 + uxtb r7, r2, ror #8 #endif - vdup.32 q1, r9 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #1 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - eor r8, r8, r7 - eor r9, r9, r8 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} -#else - strd r4, r5, [r2], #8 -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} -#else - strd r6, r7, [r2], #8 -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} #else - strd r8, r9, [r2], #8 + ubfx r7, r2, #8, #8 #endif - vdup.32 q1, r9 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #2 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - eor r8, r8, r7 - eor r9, r9, r8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r8, r2, #8 + lsr r8, r8, #24 #else - strd r4, r5, [r2], #8 + uxtb r8, r2, ror #16 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r8, r2, #16, #8 #endif + lsr r9, r2, #24 + ldrb r6, [r12, r6, lsl #2] + ldrb r7, [r12, r7, lsl #2] + ldrb r8, [r12, r8, lsl #2] + ldrb r9, [r12, r9, lsl #2] + ldr r6, [lr, r6, lsl #2] + ldr r7, [lr, r7, lsl #2] + ldr r8, [lr, r8, lsl #2] + ldr r9, [lr, r9, lsl #2] + eor r8, r8, r6, ror #16 + eor r8, r8, r7, ror #8 + eor r8, r8, r9, ror #24 + str r8, [r0], #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r3, #24 + lsr r6, r6, #24 #else - strd r8, r9, [r2], #8 + uxtb r6, r3 #endif - vdup.32 q1, r9 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #4 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - eor r8, r8, r7 - eor r9, r9, r8 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} #else - strd r4, r5, [r2], #8 + ubfx r6, r3, #0, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r3, #16 + lsr r7, r7, #24 #else - strd r6, r7, [r2], #8 + uxtb r7, r3, ror #8 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} #else - strd r8, r9, [r2], #8 + ubfx r7, r3, #8, #8 #endif - vdup.32 q1, r9 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #8 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - eor r8, r8, r7 - eor r9, r9, r8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r8, r3, #8 + lsr r8, r8, #24 #else - strd r4, r5, [r2], #8 + uxtb r8, r3, ror #16 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r8, r3, #16, #8 #endif + lsr r9, r3, #24 + ldrb r6, [r12, r6, lsl #2] + ldrb r7, [r12, r7, lsl #2] + ldrb r8, [r12, r8, lsl #2] + ldrb r9, [r12, r9, lsl #2] + ldr r6, [lr, r6, lsl #2] + ldr r7, [lr, r7, lsl #2] + ldr r8, [lr, r8, lsl #2] + ldr r9, [lr, r9, lsl #2] + eor r8, r8, r6, ror #16 + eor r8, r8, r7, ror #8 + eor r8, r8, r9, ror #24 + str r8, [r0], #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r4, #24 + lsr r6, r6, #24 #else - strd r8, r9, [r2], #8 + uxtb r6, r4 #endif - vdup.32 q1, r9 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #16 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - eor r8, r8, r7 - eor r9, r9, r8 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} #else - strd r4, r5, [r2], #8 + ubfx r6, r4, #0, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r4, #16 + lsr r7, r7, #24 #else - strd r6, r7, [r2], #8 + uxtb r7, r4, ror #8 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} #else - strd r8, r9, [r2], #8 + ubfx r7, r4, #8, #8 #endif - vdup.32 q1, r9 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #32 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - eor r8, r8, r7 - eor r9, r9, r8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r8, r4, #8 + lsr r8, r8, #24 #else - strd r4, r5, [r2], #8 + uxtb r8, r4, ror #16 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r8, r4, #16, #8 #endif + lsr r9, r4, #24 + ldrb r6, [r12, r6, lsl #2] + ldrb r7, [r12, r7, lsl #2] + ldrb r8, [r12, r8, lsl #2] + ldrb r9, [r12, r9, lsl #2] + ldr r6, [lr, r6, lsl #2] + ldr r7, [lr, r7, lsl #2] + ldr r8, [lr, r8, lsl #2] + ldr r9, [lr, r9, lsl #2] + eor r8, r8, r6, ror #16 + eor r8, r8, r7, ror #8 + eor r8, r8, r9, ror #24 + str r8, [r0], #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r5, #24 + lsr r6, r6, #24 #else - strd r8, r9, [r2], #8 + uxtb r6, r5 #endif - vdup.32 q1, r9 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #0x40 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - eor r8, r8, r7 - eor r9, r9, r8 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} #else - strd r4, r5, [r2], #8 + ubfx r6, r5, #0, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r5, #16 + lsr r7, r7, #24 #else - strd r6, r7, [r2], #8 + uxtb r7, r5, ror #8 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} #else - strd r8, r9, [r2], #8 + ubfx r7, r5, #8, #8 #endif - vdup.32 q1, r9 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #0x80 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r8, r5, #8 + lsr r8, r8, #24 #else - strd r4, r5, [r2], #8 + uxtb r8, r5, ror #16 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r8, r5, #16, #8 #endif - cmp r3, #0 - beq L_aes_set_key_arm32_crypto_done - sub r2, r2, #0xd0 - vld1.32 {q0}, [r2] - add r2, r2, #0xc0 - vld1.32 {q1}, [r2] - sub r2, r2, #0xc0 - vst1.32 {q1}, [r2] - add r2, r2, #0xc0 - vst1.32 {q0}, [r2] - sub r2, r2, #0xb0 - vld1.32 {q0}, [r2] - add r2, r2, #0xa0 - vld1.32 {q1}, [r2] - sub r2, r2, #0xa0 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0xa0 - vst1.32 {q0}, [r2] - sub r2, r2, #0x90 - vld1.32 {q0}, [r2] - add r2, r2, #0x80 - vld1.32 {q1}, [r2] - sub r2, r2, #0x80 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0x80 - vst1.32 {q0}, [r2] - sub r2, r2, #0x70 - vld1.32 {q0}, [r2] - add r2, r2, #0x60 - vld1.32 {q1}, [r2] - sub r2, r2, #0x60 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0x60 - vst1.32 {q0}, [r2] - sub r2, r2, #0x50 - vld1.32 {q0}, [r2] - add r2, r2, #0x40 - vld1.32 {q1}, [r2] - sub r2, r2, #0x40 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0x40 - vst1.32 {q0}, [r2] - sub r2, r2, #48 - vld1.32 {q0}, [r2] - add r2, r2, #32 - vld1.32 {q1}, [r2] - sub r2, r2, #32 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #32 - vst1.32 {q0}, [r2] + lsr r9, r5, #24 + ldrb r6, [r12, r6, lsl #2] + ldrb r7, [r12, r7, lsl #2] + ldrb r8, [r12, r8, lsl #2] + ldrb r9, [r12, r9, lsl #2] + ldr r6, [lr, r6, lsl #2] + ldr r7, [lr, r7, lsl #2] + ldr r8, [lr, r8, lsl #2] + ldr r9, [lr, r9, lsl #2] + eor r8, r8, r6, ror #16 + eor r8, r8, r7, ror #8 + eor r8, r8, r9, ror #24 + str r8, [r0], #4 + subs r11, r11, #1 + bne L_AES_invert_key_mix_loop + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size AES_invert_key,.-AES_invert_key +#endif /* HAVE_AES_DECRYPT */ +#ifndef __APPLE__ + .text + .type L_AES_ARM32_rcon, %object + .size L_AES_ARM32_rcon, 40 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_AES_ARM32_rcon: + .long 0x01000000,0x02000000,0x04000000,0x08000000 + .long 0x10000000,0x20000000,0x40000000,0x80000000 + .long 0x1b000000,0x36000000 + .text + .align 4 + .globl AES_set_encrypt_key + .type AES_set_encrypt_key, %function +AES_set_encrypt_key: + push {r4, r5, r6, r7, r8, lr} + adr r8, L_AES_ARM32_te + ldr r8, [r8] + adr lr, L_AES_ARM32_rcon + cmp r1, #0x80 + beq L_AES_set_encrypt_key_start_128 + cmp r1, #0xc0 + beq L_AES_set_encrypt_key_start_192 + ldr r4, [r0] + ldr r5, [r0, #4] + ldr r6, [r0, #8] + ldr r7, [r0, #12] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r2!, {r4, r5, r6, r7} + ldr r4, [r0, #16] + ldr r5, [r0, #20] + ldr r6, [r0, #24] + ldr r7, [r0, #28] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r2, {r4, r5, r6, r7} sub r2, r2, #16 - vld1.32 {q0}, [r2] - aesimc.8 q0, q0 - vst1.32 {q0}, [r2] - b L_aes_set_key_arm32_crypto_done -L_aes_set_key_arm32_crypto_start_256: - ldr r4, [r0], #4 - ldr r5, [r0], #4 - ldr r6, [r0], #4 - ldr r7, [r0], #4 - ldr r8, [r0], #4 - ldr r9, [r0], #4 - ldr r10, [r0], #4 - ldr r11, [r0], #4 + mov r12, #6 +L_AES_set_encrypt_key_loop_256: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r4, r7, #24 + lsr r4, r4, #24 #else - strd r4, r5, [r2], #8 + uxtb r4, r7 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r4, r7, #0, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r5, r7, #16 + lsr r5, r5, #24 #else - strd r8, r9, [r2], #8 + uxtb r5, r7, ror #8 +#endif +#else + ubfx r5, r7, #8, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r10, r11} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r7, #8 + lsr r6, r6, #24 #else - strd r10, r11, [r2], #8 + uxtb r6, r7, ror #16 #endif - vdup.32 q1, r11 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #1 - eor r4, r4, r12 +#else + ubfx r6, r7, #16, #8 +#endif + lsr r7, r7, #24 + ldrb r4, [r8, r4, lsl #2] + ldrb r5, [r8, r5, lsl #2] + ldrb r6, [r8, r6, lsl #2] + ldrb r7, [r8, r7, lsl #2] + eor r3, r7, r4, lsl #8 + eor r3, r3, r5, lsl #16 + eor r3, r3, r6, lsl #24 + ldm r2!, {r4, r5, r6, r7} + eor r4, r4, r3 + ldm lr!, {r3} + eor r4, r4, r3 eor r5, r5, r4 eor r6, r6, r5 eor r7, r7, r6 - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - eor r8, r8, r12 - eor r9, r9, r8 - eor r10, r10, r9 - eor r11, r11, r10 + add r2, r2, #16 + stm r2, {r4, r5, r6, r7} + sub r2, r2, #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r4, r7, #16 + lsr r4, r4, #24 #else - strd r4, r5, [r2], #8 + uxtb r4, r7, ror #8 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r4, r7, #8, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r5, r7, #8 + lsr r5, r5, #24 #else - strd r8, r9, [r2], #8 + uxtb r5, r7, ror #16 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r10, r11} #else - strd r10, r11, [r2], #8 + ubfx r5, r7, #16, #8 #endif - vdup.32 q1, r11 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #2 - eor r4, r4, r12 + lsr r6, r7, #24 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r3, r7, #24 + lsr r3, r3, #24 +#else + uxtb r3, r7 +#endif +#else + ubfx r3, r7, #0, #8 +#endif + ldrb r4, [r8, r4, lsl #2] + ldrb r6, [r8, r6, lsl #2] + ldrb r5, [r8, r5, lsl #2] + ldrb r3, [r8, r3, lsl #2] + eor r3, r3, r4, lsl #8 + eor r3, r3, r5, lsl #16 + eor r3, r3, r6, lsl #24 + ldm r2!, {r4, r5, r6, r7} + eor r4, r4, r3 eor r5, r5, r4 eor r6, r6, r5 eor r7, r7, r6 - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - eor r8, r8, r12 - eor r9, r9, r8 - eor r10, r10, r9 - eor r11, r11, r10 + add r2, r2, #16 + stm r2, {r4, r5, r6, r7} + sub r2, r2, #16 + subs r12, r12, #1 + bne L_AES_set_encrypt_key_loop_256 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r4, r7, #24 + lsr r4, r4, #24 #else - strd r4, r5, [r2], #8 + uxtb r4, r7 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r4, r7, #0, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r5, r7, #16 + lsr r5, r5, #24 #else - strd r8, r9, [r2], #8 + uxtb r5, r7, ror #8 +#endif +#else + ubfx r5, r7, #8, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r10, r11} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r7, #8 + lsr r6, r6, #24 #else - strd r10, r11, [r2], #8 + uxtb r6, r7, ror #16 #endif - vdup.32 q1, r11 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #4 - eor r4, r4, r12 +#else + ubfx r6, r7, #16, #8 +#endif + lsr r7, r7, #24 + ldrb r4, [r8, r4, lsl #2] + ldrb r5, [r8, r5, lsl #2] + ldrb r6, [r8, r6, lsl #2] + ldrb r7, [r8, r7, lsl #2] + eor r3, r7, r4, lsl #8 + eor r3, r3, r5, lsl #16 + eor r3, r3, r6, lsl #24 + ldm r2!, {r4, r5, r6, r7} + eor r4, r4, r3 + ldm lr!, {r3} + eor r4, r4, r3 eor r5, r5, r4 eor r6, r6, r5 eor r7, r7, r6 - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - eor r8, r8, r12 - eor r9, r9, r8 - eor r10, r10, r9 - eor r11, r11, r10 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} + add r2, r2, #16 + stm r2, {r4, r5, r6, r7} + sub r2, r2, #16 + b L_AES_set_encrypt_key_end +L_AES_set_encrypt_key_start_192: + ldr r4, [r0] + ldr r5, [r0, #4] + ldr r6, [r0, #8] + ldr r7, [r0, #12] + ldr r1, [r0, #20] + ldr r0, [r0, #16] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 + # REV r0, r0 + eor r3, r0, r0, ror #16 + bic r3, r3, #0xff0000 + ror r0, r0, #8 + eor r0, r0, r3, lsr #8 + # REV r1, r1 + eor r3, r1, r1, ror #16 + bic r3, r3, #0xff0000 + ror r1, r1, #8 + eor r1, r1, r3, lsr #8 #else - strd r4, r5, [r2], #8 -#endif + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 + rev r0, r0 + rev r1, r1 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r2, {r4, r5, r6, r7} #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} + str r0, [r2, #16] + str r1, [r2, #20] #else - strd r6, r7, [r2], #8 + strd r0, r1, [r2, #16] #endif + mov r7, r1 + mov r12, #7 +L_AES_set_encrypt_key_loop_192: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r0, r7, #24 + lsr r0, r0, #24 #else - strd r8, r9, [r2], #8 + uxtb r0, r7 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r10, r11} #else - strd r10, r11, [r2], #8 + ubfx r0, r7, #0, #8 #endif - vdup.32 q1, r11 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #8 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - eor r8, r8, r12 - eor r9, r9, r8 - eor r10, r10, r9 - eor r11, r11, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r1, r7, #16 + lsr r1, r1, #24 #else - strd r4, r5, [r2], #8 + uxtb r1, r7, ror #8 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r1, r7, #8, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r4, r7, #8 + lsr r4, r4, #24 #else - strd r8, r9, [r2], #8 + uxtb r4, r7, ror #16 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r10, r11} #else - strd r10, r11, [r2], #8 + ubfx r4, r7, #16, #8 #endif - vdup.32 q1, r11 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #16 - eor r4, r4, r12 + lsr r7, r7, #24 + ldrb r0, [r8, r0, lsl #2] + ldrb r1, [r8, r1, lsl #2] + ldrb r4, [r8, r4, lsl #2] + ldrb r7, [r8, r7, lsl #2] + eor r3, r7, r0, lsl #8 + eor r3, r3, r1, lsl #16 + eor r3, r3, r4, lsl #24 + ldm r2!, {r0, r1, r4, r5, r6, r7} + eor r0, r0, r3 + ldm lr!, {r3} + eor r0, r0, r3 + eor r1, r1, r0 + eor r4, r4, r1 eor r5, r5, r4 eor r6, r6, r5 eor r7, r7, r6 - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - eor r8, r8, r12 - eor r9, r9, r8 - eor r10, r10, r9 - eor r11, r11, r10 + stm r2, {r0, r1, r4, r5, r6, r7} + subs r12, r12, #1 + bne L_AES_set_encrypt_key_loop_192 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r0, r7, #24 + lsr r0, r0, #24 #else - strd r4, r5, [r2], #8 + uxtb r0, r7 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r0, r7, #0, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r1, r7, #16 + lsr r1, r1, #24 #else - strd r8, r9, [r2], #8 + uxtb r1, r7, ror #8 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r10, r11} #else - strd r10, r11, [r2], #8 + ubfx r1, r7, #8, #8 #endif - vdup.32 q1, r11 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #32 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - eor r8, r8, r12 - eor r9, r9, r8 - eor r10, r10, r9 - eor r11, r11, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r4, r7, #8 + lsr r4, r4, #24 #else - strd r4, r5, [r2], #8 + uxtb r4, r7, ror #16 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r4, r7, #16, #8 #endif + lsr r7, r7, #24 + ldrb r0, [r8, r0, lsl #2] + ldrb r1, [r8, r1, lsl #2] + ldrb r4, [r8, r4, lsl #2] + ldrb r7, [r8, r7, lsl #2] + eor r3, r7, r0, lsl #8 + eor r3, r3, r1, lsl #16 + eor r3, r3, r4, lsl #24 + ldm r2!, {r0, r1, r4, r5, r6, r7} + eor r0, r0, r3 + ldm lr!, {r3} + eor r0, r0, r3 + eor r1, r1, r0 + eor r4, r4, r1 + eor r5, r5, r4 + stm r2, {r0, r1, r4, r5} + b L_AES_set_encrypt_key_end +L_AES_set_encrypt_key_start_128: + ldr r4, [r0] + ldr r5, [r0, #4] + ldr r6, [r0, #8] + ldr r7, [r0, #12] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r2, {r4, r5, r6, r7} + mov r12, #10 +L_AES_set_encrypt_key_loop_128: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r8, r9} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r4, r7, #24 + lsr r4, r4, #24 #else - strd r8, r9, [r2], #8 + uxtb r4, r7 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r10, r11} #else - strd r10, r11, [r2], #8 + ubfx r4, r7, #0, #8 #endif - vdup.32 q1, r11 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #0x40 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r5, r7, #16 + lsr r5, r5, #24 #else - strd r4, r5, [r2], #8 + uxtb r5, r7, ror #8 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r5, r7, #8, #8 #endif - cmp r3, #0 - beq L_aes_set_key_arm32_crypto_done - sub r2, r2, #0xf0 - vld1.32 {q0}, [r2] - add r2, r2, #0xe0 - vld1.32 {q1}, [r2] - sub r2, r2, #0xe0 - vst1.32 {q1}, [r2] - add r2, r2, #0xe0 - vst1.32 {q0}, [r2] - sub r2, r2, #0xd0 - vld1.32 {q0}, [r2] - add r2, r2, #0xc0 - vld1.32 {q1}, [r2] - sub r2, r2, #0xc0 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0xc0 - vst1.32 {q0}, [r2] - sub r2, r2, #0xb0 - vld1.32 {q0}, [r2] - add r2, r2, #0xa0 - vld1.32 {q1}, [r2] - sub r2, r2, #0xa0 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0xa0 - vst1.32 {q0}, [r2] - sub r2, r2, #0x90 - vld1.32 {q0}, [r2] - add r2, r2, #0x80 - vld1.32 {q1}, [r2] - sub r2, r2, #0x80 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0x80 - vst1.32 {q0}, [r2] - sub r2, r2, #0x70 - vld1.32 {q0}, [r2] - add r2, r2, #0x60 - vld1.32 {q1}, [r2] - sub r2, r2, #0x60 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0x60 - vst1.32 {q0}, [r2] - sub r2, r2, #0x50 - vld1.32 {q0}, [r2] - add r2, r2, #0x40 - vld1.32 {q1}, [r2] - sub r2, r2, #0x40 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0x40 - vst1.32 {q0}, [r2] - sub r2, r2, #48 - vld1.32 {q0}, [r2] - add r2, r2, #32 - vld1.32 {q1}, [r2] - sub r2, r2, #32 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #32 - vst1.32 {q0}, [r2] - sub r2, r2, #16 - vld1.32 {q0}, [r2] - aesimc.8 q0, q0 - vst1.32 {q0}, [r2] - b L_aes_set_key_arm32_crypto_done -L_aes_set_key_arm32_crypto_start_128: - ldr r4, [r0], #4 - ldr r5, [r0], #4 - ldr r6, [r0], #4 - ldr r7, [r0], #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r7, #8 + lsr r6, r6, #24 #else - strd r4, r5, [r2], #8 + uxtb r6, r7, ror #16 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r6, r7, #16, #8 #endif - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #1 - eor r4, r4, r12 + lsr r7, r7, #24 + ldrb r4, [r8, r4, lsl #2] + ldrb r5, [r8, r5, lsl #2] + ldrb r6, [r8, r6, lsl #2] + ldrb r7, [r8, r7, lsl #2] + eor r3, r7, r4, lsl #8 + eor r3, r3, r5, lsl #16 + eor r3, r3, r6, lsl #24 + ldm r2!, {r4, r5, r6, r7} + eor r4, r4, r3 + ldm lr!, {r3} + eor r4, r4, r3 eor r5, r5, r4 eor r6, r6, r5 eor r7, r7, r6 + stm r2, {r4, r5, r6, r7} + subs r12, r12, #1 + bne L_AES_set_encrypt_key_loop_128 +L_AES_set_encrypt_key_end: + pop {r4, r5, r6, r7, r8, pc} + .size AES_set_encrypt_key,.-AES_set_encrypt_key +#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE + .text + .align 4 + .globl AES_encrypt_block + .type AES_encrypt_block, %function +AES_encrypt_block: + push {lr} +L_AES_encrypt_block_nr: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r8, r5, #8 + lsr r8, r8, #24 #else - strd r4, r5, [r2], #8 + uxtb r8, r5, ror #16 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r8, r5, #16, #8 #endif - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #2 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 + lsr r11, r4, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r6, #16 + lsr lr, lr, #24 #else - strd r4, r5, [r2], #8 + uxtb lr, r6, ror #8 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx lr, r6, #8, #8 #endif - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #4 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r7, #24 + lsr r2, r2, #24 #else - strd r4, r5, [r2], #8 + uxtb r2, r7 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r2, r7, #0, #8 #endif - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #8 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 + ldr r8, [r0, r8, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r9, r6, #8 + lsr r9, r9, #24 #else - strd r4, r5, [r2], #8 + uxtb r9, r6, ror #16 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} #else - strd r6, r7, [r2], #8 + ubfx r9, r6, #16, #8 #endif - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #16 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 + eor r8, r8, r11, ror #24 + lsr r11, r5, #24 + eor r8, r8, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r7, #16 + lsr lr, lr, #24 #else - strd r4, r5, [r2], #8 + uxtb lr, r7, ror #8 #endif +#else + ubfx lr, r7, #8, #8 +#endif + eor r8, r8, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r4, #24 + lsr r2, r2, #24 #else - strd r6, r7, [r2], #8 + uxtb r2, r4 #endif - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #32 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 +#else + ubfx r2, r4, #0, #8 +#endif + ldr r9, [r0, r9, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r10, r7, #8 + lsr r10, r10, #24 #else - strd r4, r5, [r2], #8 + uxtb r10, r7, ror #16 +#endif +#else + ubfx r10, r7, #16, #8 #endif + eor r9, r9, r11, ror #24 + lsr r11, r6, #24 + eor r9, r9, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r4, #16 + lsr lr, lr, #24 #else - strd r6, r7, [r2], #8 + uxtb lr, r4, ror #8 #endif - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #0x40 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 +#else + ubfx lr, r4, #8, #8 +#endif + eor r9, r9, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r5, #24 + lsr r2, r2, #24 #else - strd r4, r5, [r2], #8 + uxtb r2, r5 +#endif +#else + ubfx r2, r5, #0, #8 #endif + ldr r10, [r0, r10, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r6, #24 + lsr r6, r6, #24 #else - strd r6, r7, [r2], #8 + uxtb r6, r6 #endif - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - ror r12, r12, #8 - eor r4, r4, #0x80 - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 +#else + ubfx r6, r6, #0, #8 +#endif + eor r10, r10, r11, ror #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r11, r4, #8 + lsr r11, r11, #24 #else - strd r4, r5, [r2], #8 + uxtb r11, r4, ror #16 +#endif +#else + ubfx r11, r4, #16, #8 #endif + eor r10, r10, lr, ror #8 + lsr lr, r7, #24 + eor r10, r10, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r5, #16 + lsr r2, r2, #24 #else - strd r6, r7, [r2], #8 + uxtb r2, r5, ror #8 #endif - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - mov lr, #27 - ror r12, r12, #8 - eor r4, r4, lr - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 +#else + ubfx r2, r5, #8, #8 +#endif + ldr r6, [r0, r6, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr r2, [r0, r2, lsl #2] + eor lr, lr, r6, ror #24 + ldm r3!, {r4, r5, r6, r7} + eor r11, r11, lr, ror #24 + eor r11, r11, r2, ror #8 + # XOR in Key Schedule + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r4, r9, #8 + lsr r4, r4, #24 #else - strd r4, r5, [r2], #8 + uxtb r4, r9, ror #16 +#endif +#else + ubfx r4, r9, #16, #8 #endif + lsr r7, r8, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r10, #16 + lsr lr, lr, #24 #else - strd r6, r7, [r2], #8 + uxtb lr, r10, ror #8 +#endif +#else + ubfx lr, r10, #8, #8 #endif - vdup.32 q1, r7 - vmov.i32 q0, #0 - aese.8 q0, q1 - vmov.i32 r12, s0 - mov lr, #54 - ror r12, r12, #8 - eor r4, r4, lr - eor r4, r4, r12 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r4, r5} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r11, #24 + lsr r2, r2, #24 #else - strd r4, r5, [r2], #8 + uxtb r2, r11 +#endif +#else + ubfx r2, r11, #0, #8 #endif + ldr r4, [r0, r4, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r2!, {r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r5, r10, #8 + lsr r5, r5, #24 #else - strd r6, r7, [r2], #8 -#endif - cmp r3, #0 - beq L_aes_set_key_arm32_crypto_done - sub r2, r2, #0xb0 - vld1.32 {q0}, [r2] - add r2, r2, #0xa0 - vld1.32 {q1}, [r2] - sub r2, r2, #0xa0 - vst1.32 {q1}, [r2] - add r2, r2, #0xa0 - vst1.32 {q0}, [r2] - sub r2, r2, #0x90 - vld1.32 {q0}, [r2] - add r2, r2, #0x80 - vld1.32 {q1}, [r2] - sub r2, r2, #0x80 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0x80 - vst1.32 {q0}, [r2] - sub r2, r2, #0x70 - vld1.32 {q0}, [r2] - add r2, r2, #0x60 - vld1.32 {q1}, [r2] - sub r2, r2, #0x60 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0x60 - vst1.32 {q0}, [r2] - sub r2, r2, #0x50 - vld1.32 {q0}, [r2] - add r2, r2, #0x40 - vld1.32 {q1}, [r2] - sub r2, r2, #0x40 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #0x40 - vst1.32 {q0}, [r2] - sub r2, r2, #48 - vld1.32 {q0}, [r2] - add r2, r2, #32 - vld1.32 {q1}, [r2] - sub r2, r2, #32 - aesimc.8 q0, q0 - aesimc.8 q1, q1 - vst1.32 {q1}, [r2] - add r2, r2, #32 - vst1.32 {q0}, [r2] - sub r2, r2, #16 - vld1.32 {q0}, [r2] - aesimc.8 q0, q0 - vst1.32 {q0}, [r2] -L_aes_set_key_arm32_crypto_done: - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size AES_set_key_AARCH32,.-AES_set_key_AARCH32 -#if defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AES_CBC) - .text - .align 4 - .globl AES_encrypt_AARCH32 - .type AES_encrypt_AARCH32, %function -AES_encrypt_AARCH32: - vpush {d8-d9} - vld1.8 {q0}, [r0] - vldm r2!, {q1-q4} - aese.8 q0, q1 - aesmc.8 q0, q0 - aese.8 q0, q2 - aesmc.8 q0, q0 - aese.8 q0, q3 - aesmc.8 q0, q0 - aese.8 q0, q4 - aesmc.8 q0, q0 - vldm r2!, {q1-q4} - aese.8 q0, q1 - aesmc.8 q0, q0 - aese.8 q0, q2 - aesmc.8 q0, q0 - aese.8 q0, q3 - aesmc.8 q0, q0 - aese.8 q0, q4 - aesmc.8 q0, q0 - subs r3, r3, #10 - vld1.32 {q1-q2}, [r2]! - aese.8 q0, q1 - aesmc.8 q0, q0 - aese.8 q0, q2 - beq L_aes_encrypt_arm32_crypto_round_done - vld1.32 {q1-q2}, [r2]! - subs r3, r3, #2 - aesmc.8 q0, q0 - aese.8 q0, q1 - aesmc.8 q0, q0 - aese.8 q0, q2 - beq L_aes_encrypt_arm32_crypto_round_done - vld1.32 {q1-q2}, [r2]! - aesmc.8 q0, q0 - aese.8 q0, q1 - aesmc.8 q0, q0 - aese.8 q0, q2 -L_aes_encrypt_arm32_crypto_round_done: - vld1.32 {q1}, [r2] - veor.32 q0, q0, q1 - vst1.8 {q0}, [r1] - vpop {d8-d9} - bx lr - .size AES_encrypt_AARCH32,.-AES_encrypt_AARCH32 -#endif /* defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AES_CBC) */ -#if !defined(WC_AES_BITSLICED) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) -#ifdef HAVE_AES_DECRYPT - .text - .align 4 - .globl AES_decrypt_AARCH32 - .type AES_decrypt_AARCH32, %function -AES_decrypt_AARCH32: - vpush {d8-d9} - vld1.8 {q0}, [r0] - vldm r2!, {q1-q4} - aesd.8 q0, q1 - aesimc.8 q0, q0 - aesd.8 q0, q2 - aesimc.8 q0, q0 - aesd.8 q0, q3 - aesimc.8 q0, q0 - aesd.8 q0, q4 - aesimc.8 q0, q0 - vldm r2!, {q1-q4} - aesd.8 q0, q1 - aesimc.8 q0, q0 - aesd.8 q0, q2 - aesimc.8 q0, q0 - aesd.8 q0, q3 - aesimc.8 q0, q0 - aesd.8 q0, q4 - aesimc.8 q0, q0 - vld1.32 {q1-q2}, [r2]! - aesd.8 q0, q1 - aesimc.8 q0, q0 - aesd.8 q0, q2 - subs r3, r3, #10 - beq L_aes_decrypt_arm32_crypto_round_done - vld1.32 {q1-q2}, [r2]! - aesimc.8 q0, q0 - aesd.8 q0, q1 - aesimc.8 q0, q0 - aesd.8 q0, q2 - subs r3, r3, #2 - beq L_aes_decrypt_arm32_crypto_round_done - vld1.32 {q1-q2}, [r2]! - aesimc.8 q0, q0 - aesd.8 q0, q1 - aesimc.8 q0, q0 - aesd.8 q0, q2 -L_aes_decrypt_arm32_crypto_round_done: - vld1.32 {q1}, [r2] - veor.32 q0, q0, q1 - vst1.8 {q0}, [r1] - vpop {d8-d9} - bx lr - .size AES_decrypt_AARCH32,.-AES_decrypt_AARCH32 -#endif /* HAVE_AES_DECRYPT */ -#endif /* !defined(WC_AES_BITSLICED) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) */ -#ifdef HAVE_AES_ECB - .text - .align 4 - .globl AES_encrypt_blocks_AARCH32 - .type AES_encrypt_blocks_AARCH32, %function -AES_encrypt_blocks_AARCH32: - vpush {d8-d15} - ldr r12, [sp, #64] - vldm.32 r3!, {q0-q7} - lsr r2, r2, #4 - cmp r12, #12 - blt L_aes_encrypt_blocks_arm32_crypto_start_128 - bgt L_aes_encrypt_blocks_arm32_crypto_start_256 - # AES_ECB_192 -#ifndef NO_AES_192 - vld1.32 {q8-q9}, [r3]! - cmp r2, #1 - beq L_aes_encrypt_blocks_arm32_crypto_192_start_1 -L_aes_encrypt_blocks_arm32_crypto_192_start_4: - cmp r2, #4 - blt L_aes_encrypt_blocks_arm32_crypto_192_start_2 - vldm.8 r0!, {q12-q15} - aese.8 q12, q0 - aesmc.8 q12, q12 - aese.8 q13, q0 - aesmc.8 q13, q13 - aese.8 q14, q0 - aesmc.8 q14, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q12, q1 - aesmc.8 q12, q12 - aese.8 q13, q1 - aesmc.8 q13, q13 - aese.8 q14, q1 - aesmc.8 q14, q14 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q12, q2 - aesmc.8 q12, q12 - aese.8 q13, q2 - aesmc.8 q13, q13 - aese.8 q14, q2 - aesmc.8 q14, q14 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q12, q3 - aesmc.8 q12, q12 - aese.8 q13, q3 - aesmc.8 q13, q13 - aese.8 q14, q3 - aesmc.8 q14, q14 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q12, q4 - aesmc.8 q12, q12 - aese.8 q13, q4 - aesmc.8 q13, q13 - aese.8 q14, q4 - aesmc.8 q14, q14 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q12, q5 - aesmc.8 q12, q12 - aese.8 q13, q5 - aesmc.8 q13, q13 - aese.8 q14, q5 - aesmc.8 q14, q14 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q12, q6 - aesmc.8 q12, q12 - aese.8 q13, q6 - aesmc.8 q13, q13 - aese.8 q14, q6 - aesmc.8 q14, q14 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q12, q7 - aesmc.8 q12, q12 - aese.8 q13, q7 - aesmc.8 q13, q13 - aese.8 q14, q7 - aesmc.8 q14, q14 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q12, q8 - aesmc.8 q12, q12 - aese.8 q13, q8 - aesmc.8 q13, q13 - aese.8 q14, q8 - aesmc.8 q14, q14 - aese.8 q15, q8 - aesmc.8 q15, q15 - vld1.32 {q10}, [r3]! - aese.8 q12, q9 - aesmc.8 q12, q12 - aese.8 q13, q9 - aesmc.8 q13, q13 - aese.8 q14, q9 - aesmc.8 q14, q14 - aese.8 q15, q9 - aesmc.8 q15, q15 - vld1.32 {q11}, [r3]! - aese.8 q12, q10 - aesmc.8 q12, q12 - aese.8 q13, q10 - aesmc.8 q13, q13 - aese.8 q14, q10 - aesmc.8 q14, q14 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.32 {q10}, [r3]! - aese.8 q12, q11 - veor.32 q12, q12, q10 - aese.8 q13, q11 - veor.32 q13, q13, q10 - aese.8 q14, q11 - veor.32 q14, q14, q10 - aese.8 q15, q11 - veor.32 q15, q15, q10 - sub r3, r3, #48 - sub r2, r2, #4 - vstm.8 r1!, {q12-q15} - cmp r2, #4 - bge L_aes_encrypt_blocks_arm32_crypto_192_start_4 -L_aes_encrypt_blocks_arm32_crypto_192_start_2: - cmp r2, #2 - blt L_aes_encrypt_blocks_arm32_crypto_192_start_1 - vld1.8 {q12-q13}, [r0]! - aese.8 q12, q0 - aesmc.8 q12, q12 - aese.8 q13, q0 - aesmc.8 q13, q13 - aese.8 q12, q1 - aesmc.8 q12, q12 - aese.8 q13, q1 - aesmc.8 q13, q13 - aese.8 q12, q2 - aesmc.8 q12, q12 - aese.8 q13, q2 - aesmc.8 q13, q13 - aese.8 q12, q3 - aesmc.8 q12, q12 - aese.8 q13, q3 - aesmc.8 q13, q13 - aese.8 q12, q4 - aesmc.8 q12, q12 - aese.8 q13, q4 - aesmc.8 q13, q13 - aese.8 q12, q5 - aesmc.8 q12, q12 - aese.8 q13, q5 - aesmc.8 q13, q13 - aese.8 q12, q6 - aesmc.8 q12, q12 - aese.8 q13, q6 - aesmc.8 q13, q13 - aese.8 q12, q7 - aesmc.8 q12, q12 - aese.8 q13, q7 - aesmc.8 q13, q13 - aese.8 q12, q8 - aesmc.8 q12, q12 - aese.8 q13, q8 - aesmc.8 q13, q13 - vld1.32 {q10}, [r3]! - aese.8 q12, q9 - aesmc.8 q12, q12 - aese.8 q13, q9 - aesmc.8 q13, q13 - vld1.32 {q11}, [r3]! - aese.8 q12, q10 - aesmc.8 q12, q12 - aese.8 q13, q10 - aesmc.8 q13, q13 - vld1.32 {q10}, [r3]! - aese.8 q12, q11 - veor.32 q12, q12, q10 - aese.8 q13, q11 - veor.32 q13, q13, q10 - sub r3, r3, #48 - sub r2, r2, #2 - vst1.8 {q12-q13}, [r1]! -L_aes_encrypt_blocks_arm32_crypto_192_start_1: - cmp r2, #0 - beq L_aes_encrypt_blocks_arm32_crypto_192_done - vld1.8 {q12}, [r0]! - aese.8 q12, q0 - aesmc.8 q12, q12 - aese.8 q12, q1 - aesmc.8 q12, q12 - aese.8 q12, q2 - aesmc.8 q12, q12 - aese.8 q12, q3 - aesmc.8 q12, q12 - aese.8 q12, q4 - aesmc.8 q12, q12 - aese.8 q12, q5 - aesmc.8 q12, q12 - aese.8 q12, q6 - aesmc.8 q12, q12 - aese.8 q12, q7 - aesmc.8 q12, q12 - aese.8 q12, q8 - aesmc.8 q12, q12 - vld1.32 {q10}, [r3]! - aese.8 q12, q9 - aesmc.8 q12, q12 - vld1.32 {q11}, [r3]! - aese.8 q12, q10 - aesmc.8 q12, q12 - vld1.32 {q10}, [r3]! - aese.8 q12, q11 - veor.32 q12, q12, q10 - sub r3, r3, #48 - vst1.8 {q12}, [r1]! -L_aes_encrypt_blocks_arm32_crypto_192_done: -#endif /* !NO_AES_192 */ - b L_aes_encrypt_blocks_arm32_crypto_done - # AES_ECB_256 -L_aes_encrypt_blocks_arm32_crypto_start_256: -#ifndef NO_AES_256 - vld1.32 {q8-q9}, [r3]! - cmp r2, #1 - beq L_aes_encrypt_blocks_arm32_crypto_256_start_1 -L_aes_encrypt_blocks_arm32_crypto_256_start_4: - cmp r2, #4 - blt L_aes_encrypt_blocks_arm32_crypto_256_start_2 - vldm.8 r0!, {q12-q15} - aese.8 q12, q0 - aesmc.8 q12, q12 - aese.8 q13, q0 - aesmc.8 q13, q13 - aese.8 q14, q0 - aesmc.8 q14, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q12, q1 - aesmc.8 q12, q12 - aese.8 q13, q1 - aesmc.8 q13, q13 - aese.8 q14, q1 - aesmc.8 q14, q14 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q12, q2 - aesmc.8 q12, q12 - aese.8 q13, q2 - aesmc.8 q13, q13 - aese.8 q14, q2 - aesmc.8 q14, q14 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q12, q3 - aesmc.8 q12, q12 - aese.8 q13, q3 - aesmc.8 q13, q13 - aese.8 q14, q3 - aesmc.8 q14, q14 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q12, q4 - aesmc.8 q12, q12 - aese.8 q13, q4 - aesmc.8 q13, q13 - aese.8 q14, q4 - aesmc.8 q14, q14 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q12, q5 - aesmc.8 q12, q12 - aese.8 q13, q5 - aesmc.8 q13, q13 - aese.8 q14, q5 - aesmc.8 q14, q14 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q12, q6 - aesmc.8 q12, q12 - aese.8 q13, q6 - aesmc.8 q13, q13 - aese.8 q14, q6 - aesmc.8 q14, q14 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q12, q7 - aesmc.8 q12, q12 - aese.8 q13, q7 - aesmc.8 q13, q13 - aese.8 q14, q7 - aesmc.8 q14, q14 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q12, q8 - aesmc.8 q12, q12 - aese.8 q13, q8 - aesmc.8 q13, q13 - aese.8 q14, q8 - aesmc.8 q14, q14 - aese.8 q15, q8 - aesmc.8 q15, q15 - vld1.32 {q10}, [r3]! - aese.8 q12, q9 - aesmc.8 q12, q12 - aese.8 q13, q9 - aesmc.8 q13, q13 - aese.8 q14, q9 - aesmc.8 q14, q14 - aese.8 q15, q9 - aesmc.8 q15, q15 - vld1.32 {q11}, [r3]! - aese.8 q12, q10 - aesmc.8 q12, q12 - aese.8 q13, q10 - aesmc.8 q13, q13 - aese.8 q14, q10 - aesmc.8 q14, q14 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.32 {q10}, [r3]! - aese.8 q12, q11 - aesmc.8 q12, q12 - aese.8 q13, q11 - aesmc.8 q13, q13 - aese.8 q14, q11 - aesmc.8 q14, q14 - aese.8 q15, q11 - aesmc.8 q15, q15 - vld1.32 {q11}, [r3]! - aese.8 q12, q10 - aesmc.8 q12, q12 - aese.8 q13, q10 - aesmc.8 q13, q13 - aese.8 q14, q10 - aesmc.8 q14, q14 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.32 {q10}, [r3]! - aese.8 q12, q11 - veor.32 q12, q12, q10 - aese.8 q13, q11 - veor.32 q13, q13, q10 - aese.8 q14, q11 - veor.32 q14, q14, q10 - aese.8 q15, q11 - veor.32 q15, q15, q10 - sub r3, r3, #0x50 - sub r2, r2, #4 - vstm.8 r1!, {q12-q15} - cmp r2, #4 - bge L_aes_encrypt_blocks_arm32_crypto_256_start_4 -L_aes_encrypt_blocks_arm32_crypto_256_start_2: - cmp r2, #2 - blt L_aes_encrypt_blocks_arm32_crypto_256_start_1 - vld1.8 {q12-q13}, [r0]! - aese.8 q12, q0 - aesmc.8 q12, q12 - aese.8 q13, q0 - aesmc.8 q13, q13 - aese.8 q12, q1 - aesmc.8 q12, q12 - aese.8 q13, q1 - aesmc.8 q13, q13 - aese.8 q12, q2 - aesmc.8 q12, q12 - aese.8 q13, q2 - aesmc.8 q13, q13 - aese.8 q12, q3 - aesmc.8 q12, q12 - aese.8 q13, q3 - aesmc.8 q13, q13 - aese.8 q12, q4 - aesmc.8 q12, q12 - aese.8 q13, q4 - aesmc.8 q13, q13 - aese.8 q12, q5 - aesmc.8 q12, q12 - aese.8 q13, q5 - aesmc.8 q13, q13 - aese.8 q12, q6 - aesmc.8 q12, q12 - aese.8 q13, q6 - aesmc.8 q13, q13 - aese.8 q12, q7 - aesmc.8 q12, q12 - aese.8 q13, q7 - aesmc.8 q13, q13 - aese.8 q12, q8 - aesmc.8 q12, q12 - aese.8 q13, q8 - aesmc.8 q13, q13 - vld1.32 {q10}, [r3]! - aese.8 q12, q9 - aesmc.8 q12, q12 - aese.8 q13, q9 - aesmc.8 q13, q13 - vld1.32 {q11}, [r3]! - aese.8 q12, q10 - aesmc.8 q12, q12 - aese.8 q13, q10 - aesmc.8 q13, q13 - vld1.32 {q10}, [r3]! - aese.8 q12, q11 - aesmc.8 q12, q12 - aese.8 q13, q11 - aesmc.8 q13, q13 - vld1.32 {q11}, [r3]! - aese.8 q12, q10 - aesmc.8 q12, q12 - aese.8 q13, q10 - aesmc.8 q13, q13 - vld1.32 {q10}, [r3]! - aese.8 q12, q11 - veor.32 q12, q12, q10 - aese.8 q13, q11 - veor.32 q13, q13, q10 - sub r3, r3, #0x50 - sub r2, r2, #2 - vst1.8 {q12-q13}, [r1]! -L_aes_encrypt_blocks_arm32_crypto_256_start_1: - cmp r2, #0 - beq L_aes_encrypt_blocks_arm32_crypto_256_done - vld1.8 {q12}, [r0]! - aese.8 q12, q0 - aesmc.8 q12, q12 - aese.8 q12, q1 - aesmc.8 q12, q12 - aese.8 q12, q2 - aesmc.8 q12, q12 - aese.8 q12, q3 - aesmc.8 q12, q12 - aese.8 q12, q4 - aesmc.8 q12, q12 - aese.8 q12, q5 - aesmc.8 q12, q12 - aese.8 q12, q6 - aesmc.8 q12, q12 - aese.8 q12, q7 - aesmc.8 q12, q12 - aese.8 q12, q8 - aesmc.8 q12, q12 - vld1.32 {q10}, [r3]! - aese.8 q12, q9 - aesmc.8 q12, q12 - vld1.32 {q11}, [r3]! - aese.8 q12, q10 - aesmc.8 q12, q12 - vld1.32 {q10}, [r3]! - aese.8 q12, q11 - aesmc.8 q12, q12 - vld1.32 {q11}, [r3]! - aese.8 q12, q10 - aesmc.8 q12, q12 - vld1.32 {q10}, [r3]! - aese.8 q12, q11 - veor.32 q12, q12, q10 - sub r3, r3, #0x50 - vst1.8 {q12}, [r1]! -L_aes_encrypt_blocks_arm32_crypto_256_done: -#endif /* !NO_AES_256 */ - b L_aes_encrypt_blocks_arm32_crypto_done - # AES_ECB_128 -L_aes_encrypt_blocks_arm32_crypto_start_128: -#ifndef NO_AES_128 - vldm.32 r3!, {q8-q10} - cmp r2, #1 - beq L_aes_encrypt_blocks_arm32_crypto_128_start_1 -L_aes_encrypt_blocks_arm32_crypto_128_start_4: - cmp r2, #4 - blt L_aes_encrypt_blocks_arm32_crypto_128_start_2 - vldm.8 r0!, {q12-q15} - aese.8 q12, q0 - aesmc.8 q12, q12 - aese.8 q13, q0 - aesmc.8 q13, q13 - aese.8 q14, q0 - aesmc.8 q14, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q12, q1 - aesmc.8 q12, q12 - aese.8 q13, q1 - aesmc.8 q13, q13 - aese.8 q14, q1 - aesmc.8 q14, q14 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q12, q2 - aesmc.8 q12, q12 - aese.8 q13, q2 - aesmc.8 q13, q13 - aese.8 q14, q2 - aesmc.8 q14, q14 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q12, q3 - aesmc.8 q12, q12 - aese.8 q13, q3 - aesmc.8 q13, q13 - aese.8 q14, q3 - aesmc.8 q14, q14 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q12, q4 - aesmc.8 q12, q12 - aese.8 q13, q4 - aesmc.8 q13, q13 - aese.8 q14, q4 - aesmc.8 q14, q14 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q12, q5 - aesmc.8 q12, q12 - aese.8 q13, q5 - aesmc.8 q13, q13 - aese.8 q14, q5 - aesmc.8 q14, q14 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q12, q6 - aesmc.8 q12, q12 - aese.8 q13, q6 - aesmc.8 q13, q13 - aese.8 q14, q6 - aesmc.8 q14, q14 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q12, q7 - aesmc.8 q12, q12 - aese.8 q13, q7 - aesmc.8 q13, q13 - aese.8 q14, q7 - aesmc.8 q14, q14 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q12, q8 - aesmc.8 q12, q12 - aese.8 q13, q8 - aesmc.8 q13, q13 - aese.8 q14, q8 - aesmc.8 q14, q14 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q12, q9 - veor.32 q12, q12, q10 - aese.8 q13, q9 - veor.32 q13, q13, q10 - aese.8 q14, q9 - veor.32 q14, q14, q10 - aese.8 q15, q9 - veor.32 q15, q15, q10 - sub r2, r2, #4 - vstm.8 r1!, {q12-q15} - cmp r2, #4 - bge L_aes_encrypt_blocks_arm32_crypto_128_start_4 -L_aes_encrypt_blocks_arm32_crypto_128_start_2: - cmp r2, #2 - blt L_aes_encrypt_blocks_arm32_crypto_128_start_1 - vld1.8 {q12-q13}, [r0]! - aese.8 q12, q0 - aesmc.8 q12, q12 - aese.8 q13, q0 - aesmc.8 q13, q13 - aese.8 q12, q1 - aesmc.8 q12, q12 - aese.8 q13, q1 - aesmc.8 q13, q13 - aese.8 q12, q2 - aesmc.8 q12, q12 - aese.8 q13, q2 - aesmc.8 q13, q13 - aese.8 q12, q3 - aesmc.8 q12, q12 - aese.8 q13, q3 - aesmc.8 q13, q13 - aese.8 q12, q4 - aesmc.8 q12, q12 - aese.8 q13, q4 - aesmc.8 q13, q13 - aese.8 q12, q5 - aesmc.8 q12, q12 - aese.8 q13, q5 - aesmc.8 q13, q13 - aese.8 q12, q6 - aesmc.8 q12, q12 - aese.8 q13, q6 - aesmc.8 q13, q13 - aese.8 q12, q7 - aesmc.8 q12, q12 - aese.8 q13, q7 - aesmc.8 q13, q13 - aese.8 q12, q8 - aesmc.8 q12, q12 - aese.8 q13, q8 - aesmc.8 q13, q13 - aese.8 q12, q9 - veor.32 q12, q12, q10 - aese.8 q13, q9 - veor.32 q13, q13, q10 - sub r2, r2, #2 - vst1.8 {q12-q13}, [r1]! -L_aes_encrypt_blocks_arm32_crypto_128_start_1: - cmp r2, #0 - beq L_aes_encrypt_blocks_arm32_crypto_128_done - vld1.8 {q12}, [r0]! - aese.8 q12, q0 - aesmc.8 q12, q12 - aese.8 q12, q1 - aesmc.8 q12, q12 - aese.8 q12, q2 - aesmc.8 q12, q12 - aese.8 q12, q3 - aesmc.8 q12, q12 - aese.8 q12, q4 - aesmc.8 q12, q12 - aese.8 q12, q5 - aesmc.8 q12, q12 - aese.8 q12, q6 - aesmc.8 q12, q12 - aese.8 q12, q7 - aesmc.8 q12, q12 - aese.8 q12, q8 - aesmc.8 q12, q12 - aese.8 q12, q9 - veor.32 q12, q12, q10 - vst1.8 {q12}, [r1]! -L_aes_encrypt_blocks_arm32_crypto_128_done: -#endif /* !NO_AES_128 */ -L_aes_encrypt_blocks_arm32_crypto_done: - vpop {d8-d15} - bx lr - .size AES_encrypt_blocks_AARCH32,.-AES_encrypt_blocks_AARCH32 -#ifdef HAVE_AES_DECRYPT - .text - .align 4 - .globl AES_decrypt_blocks_AARCH32 - .type AES_decrypt_blocks_AARCH32, %function -AES_decrypt_blocks_AARCH32: - vpush {d8-d15} - ldr r12, [sp, #64] - vldm.32 r3!, {q0-q7} - lsr r2, r2, #4 - cmp r12, #12 - blt L_aes_decrypt_blocks_arm32_crypto_start_128 - bgt L_aes_decrypt_blocks_arm32_crypto_start_256 - # AES_ECB_192 -#ifndef NO_AES_192 - vld1.32 {q8-q9}, [r3]! - cmp r2, #1 - beq L_aes_decrypt_blocks_arm32_crypto_192_start_1 - cmp r2, #4 - blt L_aes_decrypt_blocks_arm32_crypto_192_start_2 -L_aes_decrypt_blocks_arm32_crypto_192_start_4: - vldm.8 r0!, {q12-q15} - aesd.8 q12, q0 - aesimc.8 q12, q12 - aesd.8 q13, q0 - aesimc.8 q13, q13 - aesd.8 q14, q0 - aesimc.8 q14, q14 - aesd.8 q15, q0 - aesimc.8 q15, q15 - aesd.8 q12, q1 - aesimc.8 q12, q12 - aesd.8 q13, q1 - aesimc.8 q13, q13 - aesd.8 q14, q1 - aesimc.8 q14, q14 - aesd.8 q15, q1 - aesimc.8 q15, q15 - aesd.8 q12, q2 - aesimc.8 q12, q12 - aesd.8 q13, q2 - aesimc.8 q13, q13 - aesd.8 q14, q2 - aesimc.8 q14, q14 - aesd.8 q15, q2 - aesimc.8 q15, q15 - aesd.8 q12, q3 - aesimc.8 q12, q12 - aesd.8 q13, q3 - aesimc.8 q13, q13 - aesd.8 q14, q3 - aesimc.8 q14, q14 - aesd.8 q15, q3 - aesimc.8 q15, q15 - aesd.8 q12, q4 - aesimc.8 q12, q12 - aesd.8 q13, q4 - aesimc.8 q13, q13 - aesd.8 q14, q4 - aesimc.8 q14, q14 - aesd.8 q15, q4 - aesimc.8 q15, q15 - aesd.8 q12, q5 - aesimc.8 q12, q12 - aesd.8 q13, q5 - aesimc.8 q13, q13 - aesd.8 q14, q5 - aesimc.8 q14, q14 - aesd.8 q15, q5 - aesimc.8 q15, q15 - aesd.8 q12, q6 - aesimc.8 q12, q12 - aesd.8 q13, q6 - aesimc.8 q13, q13 - aesd.8 q14, q6 - aesimc.8 q14, q14 - aesd.8 q15, q6 - aesimc.8 q15, q15 - aesd.8 q12, q7 - aesimc.8 q12, q12 - aesd.8 q13, q7 - aesimc.8 q13, q13 - aesd.8 q14, q7 - aesimc.8 q14, q14 - aesd.8 q15, q7 - aesimc.8 q15, q15 - aesd.8 q12, q8 - aesimc.8 q12, q12 - aesd.8 q13, q8 - aesimc.8 q13, q13 - aesd.8 q14, q8 - aesimc.8 q14, q14 - aesd.8 q15, q8 - aesimc.8 q15, q15 - vld1.32 {q10}, [r3]! - aesd.8 q12, q9 - aesimc.8 q12, q12 - aesd.8 q13, q9 - aesimc.8 q13, q13 - aesd.8 q14, q9 - aesimc.8 q14, q14 - aesd.8 q15, q9 - aesimc.8 q15, q15 - vld1.32 {q11}, [r3]! - aesd.8 q12, q10 - aesimc.8 q12, q12 - aesd.8 q13, q10 - aesimc.8 q13, q13 - aesd.8 q14, q10 - aesimc.8 q14, q14 - aesd.8 q15, q10 - aesimc.8 q15, q15 - vld1.32 {q10}, [r3]! - aesd.8 q12, q11 - veor.32 q12, q12, q10 - aesd.8 q13, q11 - veor.32 q13, q13, q10 - aesd.8 q14, q11 - veor.32 q14, q14, q10 - aesd.8 q15, q11 - veor.32 q15, q15, q10 - sub r3, r3, #48 - sub r2, r2, #4 - vstm.8 r1!, {q12-q15} - cmp r2, #4 - bge L_aes_decrypt_blocks_arm32_crypto_192_start_4 -L_aes_decrypt_blocks_arm32_crypto_192_start_2: - cmp r2, #2 - blt L_aes_decrypt_blocks_arm32_crypto_192_start_1 - vld1.8 {q12-q13}, [r0]! - aesd.8 q12, q0 - aesimc.8 q12, q12 - aesd.8 q13, q0 - aesimc.8 q13, q13 - aesd.8 q12, q1 - aesimc.8 q12, q12 - aesd.8 q13, q1 - aesimc.8 q13, q13 - aesd.8 q12, q2 - aesimc.8 q12, q12 - aesd.8 q13, q2 - aesimc.8 q13, q13 - aesd.8 q12, q3 - aesimc.8 q12, q12 - aesd.8 q13, q3 - aesimc.8 q13, q13 - aesd.8 q12, q4 - aesimc.8 q12, q12 - aesd.8 q13, q4 - aesimc.8 q13, q13 - aesd.8 q12, q5 - aesimc.8 q12, q12 - aesd.8 q13, q5 - aesimc.8 q13, q13 - aesd.8 q12, q6 - aesimc.8 q12, q12 - aesd.8 q13, q6 - aesimc.8 q13, q13 - aesd.8 q12, q7 - aesimc.8 q12, q12 - aesd.8 q13, q7 - aesimc.8 q13, q13 - aesd.8 q12, q8 - aesimc.8 q12, q12 - aesd.8 q13, q8 - aesimc.8 q13, q13 - vld1.32 {q10}, [r3]! - aesd.8 q12, q9 - aesimc.8 q12, q12 - aesd.8 q13, q9 - aesimc.8 q13, q13 - vld1.32 {q11}, [r3]! - aesd.8 q12, q10 - aesimc.8 q12, q12 - aesd.8 q13, q10 - aesimc.8 q13, q13 - vld1.32 {q10}, [r3]! - aesd.8 q12, q11 - veor.32 q12, q12, q10 - aesd.8 q13, q11 - veor.32 q13, q13, q10 - sub r3, r3, #48 - sub r2, r2, #2 - vst1.8 {q12-q13}, [r1]! -L_aes_decrypt_blocks_arm32_crypto_192_start_1: - cmp r2, #0 - beq L_aes_decrypt_blocks_arm32_crypto_192_done - vld1.8 {q12}, [r0]! - aesd.8 q12, q0 - aesimc.8 q12, q12 - aesd.8 q12, q1 - aesimc.8 q12, q12 - aesd.8 q12, q2 - aesimc.8 q12, q12 - aesd.8 q12, q3 - aesimc.8 q12, q12 - aesd.8 q12, q4 - aesimc.8 q12, q12 - aesd.8 q12, q5 - aesimc.8 q12, q12 - aesd.8 q12, q6 - aesimc.8 q12, q12 - aesd.8 q12, q7 - aesimc.8 q12, q12 - aesd.8 q12, q8 - aesimc.8 q12, q12 - vld1.32 {q10}, [r3]! - aesd.8 q12, q9 - aesimc.8 q12, q12 - vld1.32 {q11}, [r3]! - aesd.8 q12, q10 - aesimc.8 q12, q12 - vld1.32 {q10}, [r3]! - aesd.8 q12, q11 - veor.32 q12, q12, q10 - sub r3, r3, #48 - vst1.8 {q12}, [r1]! -L_aes_decrypt_blocks_arm32_crypto_192_done: -#endif /* !NO_AES_192 */ - b L_aes_decrypt_blocks_arm32_crypto_done - # AES_ECB_256 -L_aes_decrypt_blocks_arm32_crypto_start_256: -#ifndef NO_AES_256 - vld1.32 {q8-q9}, [r3]! - cmp r2, #1 - beq L_aes_decrypt_blocks_arm32_crypto_256_start_1 - cmp r2, #4 - blt L_aes_decrypt_blocks_arm32_crypto_256_start_2 -L_aes_decrypt_blocks_arm32_crypto_256_start_4: - vldm.8 r0!, {q12-q15} - aesd.8 q12, q0 - aesimc.8 q12, q12 - aesd.8 q13, q0 - aesimc.8 q13, q13 - aesd.8 q14, q0 - aesimc.8 q14, q14 - aesd.8 q15, q0 - aesimc.8 q15, q15 - aesd.8 q12, q1 - aesimc.8 q12, q12 - aesd.8 q13, q1 - aesimc.8 q13, q13 - aesd.8 q14, q1 - aesimc.8 q14, q14 - aesd.8 q15, q1 - aesimc.8 q15, q15 - aesd.8 q12, q2 - aesimc.8 q12, q12 - aesd.8 q13, q2 - aesimc.8 q13, q13 - aesd.8 q14, q2 - aesimc.8 q14, q14 - aesd.8 q15, q2 - aesimc.8 q15, q15 - aesd.8 q12, q3 - aesimc.8 q12, q12 - aesd.8 q13, q3 - aesimc.8 q13, q13 - aesd.8 q14, q3 - aesimc.8 q14, q14 - aesd.8 q15, q3 - aesimc.8 q15, q15 - aesd.8 q12, q4 - aesimc.8 q12, q12 - aesd.8 q13, q4 - aesimc.8 q13, q13 - aesd.8 q14, q4 - aesimc.8 q14, q14 - aesd.8 q15, q4 - aesimc.8 q15, q15 - aesd.8 q12, q5 - aesimc.8 q12, q12 - aesd.8 q13, q5 - aesimc.8 q13, q13 - aesd.8 q14, q5 - aesimc.8 q14, q14 - aesd.8 q15, q5 - aesimc.8 q15, q15 - aesd.8 q12, q6 - aesimc.8 q12, q12 - aesd.8 q13, q6 - aesimc.8 q13, q13 - aesd.8 q14, q6 - aesimc.8 q14, q14 - aesd.8 q15, q6 - aesimc.8 q15, q15 - aesd.8 q12, q7 - aesimc.8 q12, q12 - aesd.8 q13, q7 - aesimc.8 q13, q13 - aesd.8 q14, q7 - aesimc.8 q14, q14 - aesd.8 q15, q7 - aesimc.8 q15, q15 - aesd.8 q12, q8 - aesimc.8 q12, q12 - aesd.8 q13, q8 - aesimc.8 q13, q13 - aesd.8 q14, q8 - aesimc.8 q14, q14 - aesd.8 q15, q8 - aesimc.8 q15, q15 - vld1.32 {q10}, [r3]! - aesd.8 q12, q9 - aesimc.8 q12, q12 - aesd.8 q13, q9 - aesimc.8 q13, q13 - aesd.8 q14, q9 - aesimc.8 q14, q14 - aesd.8 q15, q9 - aesimc.8 q15, q15 - vld1.32 {q11}, [r3]! - aesd.8 q12, q10 - aesimc.8 q12, q12 - aesd.8 q13, q10 - aesimc.8 q13, q13 - aesd.8 q14, q10 - aesimc.8 q14, q14 - aesd.8 q15, q10 - aesimc.8 q15, q15 - vld1.32 {q10}, [r3]! - aesd.8 q12, q11 - aesimc.8 q12, q12 - aesd.8 q13, q11 - aesimc.8 q13, q13 - aesd.8 q14, q11 - aesimc.8 q14, q14 - aesd.8 q15, q11 - aesimc.8 q15, q15 - vld1.32 {q11}, [r3]! - aesd.8 q12, q10 - aesimc.8 q12, q12 - aesd.8 q13, q10 - aesimc.8 q13, q13 - aesd.8 q14, q10 - aesimc.8 q14, q14 - aesd.8 q15, q10 - aesimc.8 q15, q15 - vld1.32 {q10}, [r3]! - aesd.8 q12, q11 - veor.32 q12, q12, q10 - aesd.8 q13, q11 - veor.32 q13, q13, q10 - aesd.8 q14, q11 - veor.32 q14, q14, q10 - aesd.8 q15, q11 - veor.32 q15, q15, q10 - sub r3, r3, #0x50 - sub r2, r2, #4 - vstm.8 r1!, {q12-q15} - cmp r2, #4 - bge L_aes_decrypt_blocks_arm32_crypto_256_start_4 -L_aes_decrypt_blocks_arm32_crypto_256_start_2: - cmp r2, #2 - blt L_aes_decrypt_blocks_arm32_crypto_256_start_1 - vld1.8 {q12-q13}, [r0]! - aesd.8 q12, q0 - aesimc.8 q12, q12 - aesd.8 q13, q0 - aesimc.8 q13, q13 - aesd.8 q12, q1 - aesimc.8 q12, q12 - aesd.8 q13, q1 - aesimc.8 q13, q13 - aesd.8 q12, q2 - aesimc.8 q12, q12 - aesd.8 q13, q2 - aesimc.8 q13, q13 - aesd.8 q12, q3 - aesimc.8 q12, q12 - aesd.8 q13, q3 - aesimc.8 q13, q13 - aesd.8 q12, q4 - aesimc.8 q12, q12 - aesd.8 q13, q4 - aesimc.8 q13, q13 - aesd.8 q12, q5 - aesimc.8 q12, q12 - aesd.8 q13, q5 - aesimc.8 q13, q13 - aesd.8 q12, q6 - aesimc.8 q12, q12 - aesd.8 q13, q6 - aesimc.8 q13, q13 - aesd.8 q12, q7 - aesimc.8 q12, q12 - aesd.8 q13, q7 - aesimc.8 q13, q13 - aesd.8 q12, q8 - aesimc.8 q12, q12 - aesd.8 q13, q8 - aesimc.8 q13, q13 - vld1.32 {q10}, [r3]! - aesd.8 q12, q9 - aesimc.8 q12, q12 - aesd.8 q13, q9 - aesimc.8 q13, q13 - vld1.32 {q11}, [r3]! - aesd.8 q12, q10 - aesimc.8 q12, q12 - aesd.8 q13, q10 - aesimc.8 q13, q13 - vld1.32 {q10}, [r3]! - aesd.8 q12, q11 - aesimc.8 q12, q12 - aesd.8 q13, q11 - aesimc.8 q13, q13 - vld1.32 {q11}, [r3]! - aesd.8 q12, q10 - aesimc.8 q12, q12 - aesd.8 q13, q10 - aesimc.8 q13, q13 - vld1.32 {q10}, [r3]! - aesd.8 q12, q11 - veor.32 q12, q12, q10 - aesd.8 q13, q11 - veor.32 q13, q13, q10 - sub r3, r3, #0x50 - sub r2, r2, #2 - vst1.8 {q12-q13}, [r1]! -L_aes_decrypt_blocks_arm32_crypto_256_start_1: - cmp r2, #0 - beq L_aes_decrypt_blocks_arm32_crypto_256_done - vld1.8 {q12}, [r0]! - aesd.8 q12, q0 - aesimc.8 q12, q12 - aesd.8 q12, q1 - aesimc.8 q12, q12 - aesd.8 q12, q2 - aesimc.8 q12, q12 - aesd.8 q12, q3 - aesimc.8 q12, q12 - aesd.8 q12, q4 - aesimc.8 q12, q12 - aesd.8 q12, q5 - aesimc.8 q12, q12 - aesd.8 q12, q6 - aesimc.8 q12, q12 - aesd.8 q12, q7 - aesimc.8 q12, q12 - aesd.8 q12, q8 - aesimc.8 q12, q12 - vld1.32 {q10}, [r3]! - aesd.8 q12, q9 - aesimc.8 q12, q12 - vld1.32 {q11}, [r3]! - aesd.8 q12, q10 - aesimc.8 q12, q12 - vld1.32 {q10}, [r3]! - aesd.8 q12, q11 - aesimc.8 q12, q12 - vld1.32 {q11}, [r3]! - aesd.8 q12, q10 - aesimc.8 q12, q12 - vld1.32 {q10}, [r3]! - aesd.8 q12, q11 - veor.32 q12, q12, q10 - sub r3, r3, #0x50 - vst1.8 {q12}, [r1]! -L_aes_decrypt_blocks_arm32_crypto_256_done: -#endif /* !NO_AES_256 */ - b L_aes_decrypt_blocks_arm32_crypto_done - # AES_ECB_128 -L_aes_decrypt_blocks_arm32_crypto_start_128: -#ifndef NO_AES_128 - vldm.32 r3!, {q8-q10} - cmp r2, #1 - beq L_aes_decrypt_blocks_arm32_crypto_128_start_1 - cmp r2, #4 - blt L_aes_decrypt_blocks_arm32_crypto_128_start_2 -L_aes_decrypt_blocks_arm32_crypto_128_start_4: - vldm.8 r0!, {q12-q15} - aesd.8 q12, q0 - aesimc.8 q12, q12 - aesd.8 q13, q0 - aesimc.8 q13, q13 - aesd.8 q14, q0 - aesimc.8 q14, q14 - aesd.8 q15, q0 - aesimc.8 q15, q15 - aesd.8 q12, q1 - aesimc.8 q12, q12 - aesd.8 q13, q1 - aesimc.8 q13, q13 - aesd.8 q14, q1 - aesimc.8 q14, q14 - aesd.8 q15, q1 - aesimc.8 q15, q15 - aesd.8 q12, q2 - aesimc.8 q12, q12 - aesd.8 q13, q2 - aesimc.8 q13, q13 - aesd.8 q14, q2 - aesimc.8 q14, q14 - aesd.8 q15, q2 - aesimc.8 q15, q15 - aesd.8 q12, q3 - aesimc.8 q12, q12 - aesd.8 q13, q3 - aesimc.8 q13, q13 - aesd.8 q14, q3 - aesimc.8 q14, q14 - aesd.8 q15, q3 - aesimc.8 q15, q15 - aesd.8 q12, q4 - aesimc.8 q12, q12 - aesd.8 q13, q4 - aesimc.8 q13, q13 - aesd.8 q14, q4 - aesimc.8 q14, q14 - aesd.8 q15, q4 - aesimc.8 q15, q15 - aesd.8 q12, q5 - aesimc.8 q12, q12 - aesd.8 q13, q5 - aesimc.8 q13, q13 - aesd.8 q14, q5 - aesimc.8 q14, q14 - aesd.8 q15, q5 - aesimc.8 q15, q15 - aesd.8 q12, q6 - aesimc.8 q12, q12 - aesd.8 q13, q6 - aesimc.8 q13, q13 - aesd.8 q14, q6 - aesimc.8 q14, q14 - aesd.8 q15, q6 - aesimc.8 q15, q15 - aesd.8 q12, q7 - aesimc.8 q12, q12 - aesd.8 q13, q7 - aesimc.8 q13, q13 - aesd.8 q14, q7 - aesimc.8 q14, q14 - aesd.8 q15, q7 - aesimc.8 q15, q15 - aesd.8 q12, q8 - aesimc.8 q12, q12 - aesd.8 q13, q8 - aesimc.8 q13, q13 - aesd.8 q14, q8 - aesimc.8 q14, q14 - aesd.8 q15, q8 - aesimc.8 q15, q15 - aesd.8 q12, q9 - veor.32 q12, q12, q10 - aesd.8 q13, q9 - veor.32 q13, q13, q10 - aesd.8 q14, q9 - veor.32 q14, q14, q10 - aesd.8 q15, q9 - veor.32 q15, q15, q10 - sub r2, r2, #4 - vstm.8 r1!, {q12-q15} - cmp r2, #4 - bge L_aes_decrypt_blocks_arm32_crypto_128_start_4 -L_aes_decrypt_blocks_arm32_crypto_128_start_2: - cmp r2, #2 - blt L_aes_decrypt_blocks_arm32_crypto_128_start_1 - vld1.8 {q12-q13}, [r0]! - aesd.8 q12, q0 - aesimc.8 q12, q12 - aesd.8 q13, q0 - aesimc.8 q13, q13 - aesd.8 q12, q1 - aesimc.8 q12, q12 - aesd.8 q13, q1 - aesimc.8 q13, q13 - aesd.8 q12, q2 - aesimc.8 q12, q12 - aesd.8 q13, q2 - aesimc.8 q13, q13 - aesd.8 q12, q3 - aesimc.8 q12, q12 - aesd.8 q13, q3 - aesimc.8 q13, q13 - aesd.8 q12, q4 - aesimc.8 q12, q12 - aesd.8 q13, q4 - aesimc.8 q13, q13 - aesd.8 q12, q5 - aesimc.8 q12, q12 - aesd.8 q13, q5 - aesimc.8 q13, q13 - aesd.8 q12, q6 - aesimc.8 q12, q12 - aesd.8 q13, q6 - aesimc.8 q13, q13 - aesd.8 q12, q7 - aesimc.8 q12, q12 - aesd.8 q13, q7 - aesimc.8 q13, q13 - aesd.8 q12, q8 - aesimc.8 q12, q12 - aesd.8 q13, q8 - aesimc.8 q13, q13 - aesd.8 q12, q9 - veor.32 q12, q12, q10 - aesd.8 q13, q9 - veor.32 q13, q13, q10 - sub r2, r2, #2 - vst1.8 {q12-q13}, [r1]! -L_aes_decrypt_blocks_arm32_crypto_128_start_1: - cmp r2, #0 - beq L_aes_decrypt_blocks_arm32_crypto_128_done - vld1.8 {q12}, [r0]! - aesd.8 q12, q0 - aesimc.8 q12, q12 - aesd.8 q12, q1 - aesimc.8 q12, q12 - aesd.8 q12, q2 - aesimc.8 q12, q12 - aesd.8 q12, q3 - aesimc.8 q12, q12 - aesd.8 q12, q4 - aesimc.8 q12, q12 - aesd.8 q12, q5 - aesimc.8 q12, q12 - aesd.8 q12, q6 - aesimc.8 q12, q12 - aesd.8 q12, q7 - aesimc.8 q12, q12 - aesd.8 q12, q8 - aesimc.8 q12, q12 - aesd.8 q12, q9 - veor.32 q12, q12, q10 - vst1.8 {q12}, [r1]! -L_aes_decrypt_blocks_arm32_crypto_128_done: -#endif /* !NO_AES_128 */ -L_aes_decrypt_blocks_arm32_crypto_done: - vpop {d8-d15} - bx lr - .size AES_decrypt_blocks_AARCH32,.-AES_decrypt_blocks_AARCH32 -#endif /* HAVE_AES_DECRYPT */ -#endif /* HAVE_AES_ECB */ -#ifdef HAVE_AES_CBC - .text - .align 4 - .globl AES_CBC_encrypt_AARCH32 - .type AES_CBC_encrypt_AARCH32, %function -AES_CBC_encrypt_AARCH32: - push {lr} - vpush {d8-d15} - ldr r12, [sp, #68] - ldr lr, [sp, #72] - vldm.32 r12!, {q0-q7} - vld1.32 {q15}, [r3] - subs lr, lr, #12 - lsr r2, r2, #4 - blt L_aes_cbc_encrypt_arm32_crypto_start_128 - bgt L_aes_cbc_encrypt_arm32_crypto_start_256 - # AES_CBC_192 -#ifndef NO_AES_192 - vld1.8 {q14}, [r0]! - vldm.32 r12!, {q8-q12} - cmp r2, #1 - beq L_aes_cbc_encrypt_arm32_crypto_192_start_1 - cmp r2, #4 - blt L_aes_cbc_encrypt_arm32_crypto_192_start_2 -L_aes_cbc_encrypt_arm32_crypto_192_start_4: - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q11 - veor.32 q15, q15, q12 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q11 - veor.32 q15, q15, q12 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q11 - veor.32 q15, q15, q12 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - subs r2, r2, #4 - aese.8 q15, q10 - aesmc.8 q15, q15 - aese.8 q15, q11 - veor.32 q15, q15, q12 - beq L_aes_cbc_encrypt_arm32_crypto_192_done - vld1.8 {q14}, [r0]! - cmp r2, #4 - vst1.8 {q15}, [r1]! - bge L_aes_cbc_encrypt_arm32_crypto_192_start_4 - cmp r2, #2 - blt L_aes_cbc_encrypt_arm32_crypto_192_start_1 -L_aes_cbc_encrypt_arm32_crypto_192_start_2: - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q11 - veor.32 q15, q15, q12 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - subs r2, r2, #2 - aese.8 q15, q10 - aesmc.8 q15, q15 - aese.8 q15, q11 - veor.32 q15, q15, q12 - beq L_aes_cbc_encrypt_arm32_crypto_192_done - vld1.8 {q14}, [r0]! - vst1.8 {q15}, [r1]! -L_aes_cbc_encrypt_arm32_crypto_192_start_1: - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - aese.8 q15, q11 - veor.32 q15, q15, q12 -L_aes_cbc_encrypt_arm32_crypto_192_done: - vst1.8 {q15}, [r1]! -#endif /* !NO_AES_192 */ - b L_aes_cbc_encrypt_arm32_crypto_done - # AES_CBC_256 -L_aes_cbc_encrypt_arm32_crypto_start_256: -#ifndef NO_AES_256 - vld1.8 {q14}, [r0]! - vldm.32 r12!, {q8-q11} - add r12, r12, #16 - vld1.32 {q12-q13}, [r12] - sub r12, r12, #16 - cmp r2, #1 - beq L_aes_cbc_encrypt_arm32_crypto_256_start_1 - cmp r2, #4 - blt L_aes_cbc_encrypt_arm32_crypto_256_start_2 -L_aes_cbc_encrypt_arm32_crypto_256_start_4: - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.32 {q14}, [r12] - aese.8 q15, q11 - aesmc.8 q15, q15 - aese.8 q15, q14 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q12 - veor.32 q15, q15, q13 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.32 {q14}, [r12] - aese.8 q15, q11 - aesmc.8 q15, q15 - aese.8 q15, q14 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q12 - veor.32 q15, q15, q13 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.32 {q14}, [r12] - aese.8 q15, q11 - aesmc.8 q15, q15 - aese.8 q15, q14 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q12 - veor.32 q15, q15, q13 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.32 {q14}, [r12] - aese.8 q15, q11 - aesmc.8 q15, q15 - subs r2, r2, #4 - aese.8 q15, q14 - aesmc.8 q15, q15 - aese.8 q15, q12 - veor.32 q15, q15, q13 - beq L_aes_cbc_encrypt_arm32_crypto_256_done - vld1.8 {q14}, [r0]! - cmp r2, #4 - vst1.8 {q15}, [r1]! - bge L_aes_cbc_encrypt_arm32_crypto_256_start_4 - cmp r2, #2 - blt L_aes_cbc_encrypt_arm32_crypto_256_start_1 -L_aes_cbc_encrypt_arm32_crypto_256_start_2: - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.32 {q14}, [r12] - aese.8 q15, q11 - aesmc.8 q15, q15 - aese.8 q15, q14 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q12 - veor.32 q15, q15, q13 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.32 {q14}, [r12] - aese.8 q15, q11 - aesmc.8 q15, q15 - subs r2, r2, #2 - aese.8 q15, q14 - aesmc.8 q15, q15 - aese.8 q15, q12 - veor.32 q15, q15, q13 - beq L_aes_cbc_encrypt_arm32_crypto_256_done - vld1.8 {q14}, [r0]! - vst1.8 {q15}, [r1]! -L_aes_cbc_encrypt_arm32_crypto_256_start_1: - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - aesmc.8 q15, q15 - aese.8 q15, q10 - aesmc.8 q15, q15 - vld1.32 {q14}, [r12] - aese.8 q15, q11 - aesmc.8 q15, q15 - aese.8 q15, q14 - aesmc.8 q15, q15 - aese.8 q15, q12 - veor.32 q15, q15, q13 -L_aes_cbc_encrypt_arm32_crypto_256_done: - vst1.8 {q15}, [r1]! -#endif /* !NO_AES_256 */ - b L_aes_cbc_encrypt_arm32_crypto_done - # AES_CBC_128 -L_aes_cbc_encrypt_arm32_crypto_start_128: -#ifndef NO_AES_128 - vld1.8 {q14}, [r0]! - vldm.32 r12!, {q8-q10} - cmp r2, #1 - beq L_aes_cbc_encrypt_arm32_crypto_128_start_1 - cmp r2, #4 - blt L_aes_cbc_encrypt_arm32_crypto_128_start_2 -L_aes_cbc_encrypt_arm32_crypto_128_start_4: - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q9 - veor.32 q15, q15, q10 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q9 - veor.32 q15, q15, q10 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q9 - veor.32 q15, q15, q10 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - subs r2, r2, #4 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - veor.32 q15, q15, q10 - beq L_aes_cbc_encrypt_arm32_crypto_128_done - vld1.8 {q14}, [r0]! - cmp r2, #4 - vst1.8 {q15}, [r1]! - bge L_aes_cbc_encrypt_arm32_crypto_128_start_4 - cmp r2, #2 - blt L_aes_cbc_encrypt_arm32_crypto_128_start_1 -L_aes_cbc_encrypt_arm32_crypto_128_start_2: - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - vld1.8 {q14}, [r0]! - aese.8 q15, q9 - veor.32 q15, q15, q10 - vst1.8 {q15}, [r1]! - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - subs r2, r2, #2 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - veor.32 q15, q15, q10 - beq L_aes_cbc_encrypt_arm32_crypto_128_done - vld1.8 {q14}, [r0]! - vst1.8 {q15}, [r1]! -L_aes_cbc_encrypt_arm32_crypto_128_start_1: - veor.32 q15, q15, q14 - aese.8 q15, q0 - aesmc.8 q15, q15 - aese.8 q15, q1 - aesmc.8 q15, q15 - aese.8 q15, q2 - aesmc.8 q15, q15 - aese.8 q15, q3 - aesmc.8 q15, q15 - aese.8 q15, q4 - aesmc.8 q15, q15 - aese.8 q15, q5 - aesmc.8 q15, q15 - aese.8 q15, q6 - aesmc.8 q15, q15 - aese.8 q15, q7 - aesmc.8 q15, q15 - aese.8 q15, q8 - aesmc.8 q15, q15 - aese.8 q15, q9 - veor.32 q15, q15, q10 -L_aes_cbc_encrypt_arm32_crypto_128_done: - vst1.8 {q15}, [r1]! -#endif /* !NO_AES_128 */ -L_aes_cbc_encrypt_arm32_crypto_done: - vst1.32 {q15}, [r3] - vpop {d8-d15} - pop {pc} - .size AES_CBC_encrypt_AARCH32,.-AES_CBC_encrypt_AARCH32 -#ifdef HAVE_AES_DECRYPT - .text - .align 4 - .globl AES_CBC_decrypt_AARCH32 - .type AES_CBC_decrypt_AARCH32, %function -AES_CBC_decrypt_AARCH32: - push {lr} - vpush {d8-d15} - ldr r12, [sp, #68] - ldr lr, [sp, #72] - vldm.32 r12!, {q0-q7} - vld1.32 {q13}, [r3] - lsr r2, r2, #4 - cmp lr, #12 - blt L_aes_cbc_decrypt_blocks_arm32_crypto_start_128 - bgt L_aes_cbc_decrypt_blocks_arm32_crypto_start_256 - # AES_CBC_192 -#ifndef NO_AES_192 - vld1.32 {q8}, [r12]! - cmp r2, #1 - beq L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_1 -L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_2: - vld1.8 {q14-q15}, [r0]! - vmov q11, q13 - vmov q12, q14 - vmov q13, q15 - aesd.8 q14, q0 - aesimc.8 q14, q14 - aesd.8 q15, q0 - aesimc.8 q15, q15 - aesd.8 q14, q1 - aesimc.8 q14, q14 - aesd.8 q15, q1 - aesimc.8 q15, q15 - aesd.8 q14, q2 - aesimc.8 q14, q14 - aesd.8 q15, q2 - aesimc.8 q15, q15 - aesd.8 q14, q3 - aesimc.8 q14, q14 - aesd.8 q15, q3 - aesimc.8 q15, q15 - aesd.8 q14, q4 - aesimc.8 q14, q14 - aesd.8 q15, q4 - aesimc.8 q15, q15 - aesd.8 q14, q5 - aesimc.8 q14, q14 - aesd.8 q15, q5 - aesimc.8 q15, q15 - aesd.8 q14, q6 - aesimc.8 q14, q14 - aesd.8 q15, q6 - aesimc.8 q15, q15 - aesd.8 q14, q7 - aesimc.8 q14, q14 - aesd.8 q15, q7 - aesimc.8 q15, q15 - vld1.32 {q9}, [r12]! - aesd.8 q14, q8 - aesimc.8 q14, q14 - aesd.8 q15, q8 - aesimc.8 q15, q15 - vld1.32 {q10}, [r12]! - aesd.8 q14, q9 - aesimc.8 q14, q14 - aesd.8 q15, q9 - aesimc.8 q15, q15 - vld1.32 {q9}, [r12]! - aesd.8 q14, q10 - aesimc.8 q14, q14 - aesd.8 q15, q10 - aesimc.8 q15, q15 - vld1.32 {q10}, [r12] - aesd.8 q14, q9 - aesd.8 q15, q9 - sub r2, r2, #2 - veor.32 q14, q14, q10 - veor.32 q15, q15, q10 - cmp r2, #1 - veor.32 q14, q14, q11 - veor.32 q15, q15, q12 - vst1.8 {q14-q15}, [r1]! - sub r12, r12, #48 - blt L_aes_cbc_decrypt_blocks_arm32_crypto_192_done - bgt L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_2 -L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_1: - vld1.8 {q14}, [r0]! - vmov q11, q13 - vmov q13, q14 - aesd.8 q14, q0 - aesimc.8 q14, q14 - aesd.8 q14, q1 - aesimc.8 q14, q14 - aesd.8 q14, q2 - aesimc.8 q14, q14 - aesd.8 q14, q3 - aesimc.8 q14, q14 - aesd.8 q14, q4 - aesimc.8 q14, q14 - aesd.8 q14, q5 - aesimc.8 q14, q14 - aesd.8 q14, q6 - aesimc.8 q14, q14 - aesd.8 q14, q7 - aesimc.8 q14, q14 - vld1.32 {q9}, [r12]! - aesd.8 q14, q8 - aesimc.8 q14, q14 - vld1.32 {q10}, [r12]! - aesd.8 q14, q9 - aesimc.8 q14, q14 - vld1.32 {q9}, [r12]! - aesd.8 q14, q10 - aesimc.8 q14, q14 - vld1.32 {q10}, [r12] - aesd.8 q14, q9 - veor.32 q14, q14, q10 - veor.32 q14, q14, q11 - vst1.8 {q14}, [r1]! -L_aes_cbc_decrypt_blocks_arm32_crypto_192_done: -#endif /* !NO_AES_192 */ - b L_aes_cbc_decrypt_blocks_arm32_crypto_done - # AES_CBC_256 -L_aes_cbc_decrypt_blocks_arm32_crypto_start_256: -#ifndef NO_AES_256 - vld1.32 {q8}, [r12]! - cmp r2, #1 - beq L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_1 -L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_2: - vld1.8 {q14-q15}, [r0]! - vmov q11, q13 - vmov q12, q14 - vmov q13, q15 - aesd.8 q14, q0 - aesimc.8 q14, q14 - aesd.8 q15, q0 - aesimc.8 q15, q15 - aesd.8 q14, q1 - aesimc.8 q14, q14 - aesd.8 q15, q1 - aesimc.8 q15, q15 - aesd.8 q14, q2 - aesimc.8 q14, q14 - aesd.8 q15, q2 - aesimc.8 q15, q15 - aesd.8 q14, q3 - aesimc.8 q14, q14 - aesd.8 q15, q3 - aesimc.8 q15, q15 - aesd.8 q14, q4 - aesimc.8 q14, q14 - aesd.8 q15, q4 - aesimc.8 q15, q15 - aesd.8 q14, q5 - aesimc.8 q14, q14 - aesd.8 q15, q5 - aesimc.8 q15, q15 - aesd.8 q14, q6 - aesimc.8 q14, q14 - aesd.8 q15, q6 - aesimc.8 q15, q15 - aesd.8 q14, q7 - aesimc.8 q14, q14 - aesd.8 q15, q7 - aesimc.8 q15, q15 - vld1.32 {q9}, [r12]! - aesd.8 q14, q8 - aesimc.8 q14, q14 - aesd.8 q15, q8 - aesimc.8 q15, q15 - vld1.32 {q10}, [r12]! - aesd.8 q14, q9 - aesimc.8 q14, q14 - aesd.8 q15, q9 - aesimc.8 q15, q15 - vld1.32 {q9}, [r12]! - aesd.8 q14, q10 - aesimc.8 q14, q14 - aesd.8 q15, q10 - aesimc.8 q15, q15 - vld1.32 {q10}, [r12]! - aesd.8 q14, q9 - aesimc.8 q14, q14 - aesd.8 q15, q9 - aesimc.8 q15, q15 - vld1.32 {q9}, [r12]! - aesd.8 q14, q10 - aesimc.8 q14, q14 - aesd.8 q15, q10 - aesimc.8 q15, q15 - vld1.32 {q10}, [r12] - aesd.8 q14, q9 - aesd.8 q15, q9 - sub r2, r2, #2 - veor.32 q14, q14, q10 - veor.32 q15, q15, q10 - cmp r2, #1 - veor.32 q14, q14, q11 - veor.32 q15, q15, q12 - vst1.8 {q14-q15}, [r1]! - sub r12, r12, #0x50 - blt L_aes_cbc_decrypt_blocks_arm32_crypto_256_done - bgt L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_2 -L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_1: - vld1.8 {q14}, [r0]! - vmov q11, q13 - vmov q13, q14 - aesd.8 q14, q0 - aesimc.8 q14, q14 - aesd.8 q14, q1 - aesimc.8 q14, q14 - aesd.8 q14, q2 - aesimc.8 q14, q14 - aesd.8 q14, q3 - aesimc.8 q14, q14 - aesd.8 q14, q4 - aesimc.8 q14, q14 - aesd.8 q14, q5 - aesimc.8 q14, q14 - aesd.8 q14, q6 - aesimc.8 q14, q14 - aesd.8 q14, q7 - aesimc.8 q14, q14 - vld1.32 {q9}, [r12]! - aesd.8 q14, q8 - aesimc.8 q14, q14 - vld1.32 {q10}, [r12]! - aesd.8 q14, q9 - aesimc.8 q14, q14 - vld1.32 {q9}, [r12]! - aesd.8 q14, q10 - aesimc.8 q14, q14 - vld1.32 {q10}, [r12]! - aesd.8 q14, q9 - aesimc.8 q14, q14 - vld1.32 {q9}, [r12]! - aesd.8 q14, q10 - aesimc.8 q14, q14 - vld1.32 {q10}, [r12] - aesd.8 q14, q9 - veor.32 q14, q14, q10 - veor.32 q14, q14, q11 - vst1.8 {q14}, [r1]! -L_aes_cbc_decrypt_blocks_arm32_crypto_256_done: -#endif /* !NO_AES_256 */ - b L_aes_cbc_decrypt_blocks_arm32_crypto_done - # AES_CBC_128 -L_aes_cbc_decrypt_blocks_arm32_crypto_start_128: -#ifndef NO_AES_128 - vldm.32 r12!, {q8-q10} - cmp r2, #1 - beq L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_1 -L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_2: - vld1.8 {q14-q15}, [r0]! - vmov q11, q13 - vmov q12, q14 - vmov q13, q15 - aesd.8 q14, q0 - aesimc.8 q14, q14 - aesd.8 q15, q0 - aesimc.8 q15, q15 - aesd.8 q14, q1 - aesimc.8 q14, q14 - aesd.8 q15, q1 - aesimc.8 q15, q15 - aesd.8 q14, q2 - aesimc.8 q14, q14 - aesd.8 q15, q2 - aesimc.8 q15, q15 - aesd.8 q14, q3 - aesimc.8 q14, q14 - aesd.8 q15, q3 - aesimc.8 q15, q15 - aesd.8 q14, q4 - aesimc.8 q14, q14 - aesd.8 q15, q4 - aesimc.8 q15, q15 - aesd.8 q14, q5 - aesimc.8 q14, q14 - aesd.8 q15, q5 - aesimc.8 q15, q15 - aesd.8 q14, q6 - aesimc.8 q14, q14 - aesd.8 q15, q6 - aesimc.8 q15, q15 - aesd.8 q14, q7 - aesimc.8 q14, q14 - aesd.8 q15, q7 - aesimc.8 q15, q15 - aesd.8 q14, q8 - aesimc.8 q14, q14 - aesd.8 q15, q8 - aesimc.8 q15, q15 - aesd.8 q14, q9 - aesd.8 q15, q9 - sub r2, r2, #2 - veor.32 q14, q14, q10 - veor.32 q15, q15, q10 - cmp r2, #1 - veor.32 q14, q14, q11 - veor.32 q15, q15, q12 - vst1.8 {q14-q15}, [r1]! - blt L_aes_cbc_decrypt_blocks_arm32_crypto_128_done - bgt L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_2 -L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_1: - vld1.8 {q14}, [r0]! - vmov q11, q13 - vmov q13, q14 - aesd.8 q14, q0 - aesimc.8 q14, q14 - aesd.8 q14, q1 - aesimc.8 q14, q14 - aesd.8 q14, q2 - aesimc.8 q14, q14 - aesd.8 q14, q3 - aesimc.8 q14, q14 - aesd.8 q14, q4 - aesimc.8 q14, q14 - aesd.8 q14, q5 - aesimc.8 q14, q14 - aesd.8 q14, q6 - aesimc.8 q14, q14 - aesd.8 q14, q7 - aesimc.8 q14, q14 - aesd.8 q14, q8 - aesimc.8 q14, q14 - aesd.8 q14, q9 - veor.32 q14, q14, q10 - veor.32 q14, q14, q11 - vst1.8 {q14}, [r1]! -L_aes_cbc_decrypt_blocks_arm32_crypto_128_done: -#endif /* !NO_AES_128 */ -L_aes_cbc_decrypt_blocks_arm32_crypto_done: - vst1.32 {q13}, [r3] - vpop {d8-d15} - pop {pc} - .size AES_CBC_decrypt_AARCH32,.-AES_CBC_decrypt_AARCH32 -#endif /* HAVE_AES_DECRYPT */ -#endif /* HAVE_AES_CBC */ -#ifdef WOLFSSL_AES_COUNTER - .text - .align 4 - .globl AES_CTR_encrypt_AARCH32 - .type AES_CTR_encrypt_AARCH32, %function -AES_CTR_encrypt_AARCH32: - push {r4, r5, r6, r7, r8, lr} - vpush {d8-d15} - vld1.32 {q0}, [r3] - ldr r12, [sp, #88] - vrev32.8 q2, q0 - lsr r4, r2, #4 - vmov r5, r6, d4 - and r2, r2, #15 - vmov r7, r8, d5 - vldm.32 r12!, {q3-q10} - ldr lr, [sp, #100] - cmp lr, #12 - blt L_aes_ctr_encrypt_arm32_crypto_start_128 - bgt L_aes_ctr_encrypt_arm32_crypto_start_256 - # AES_CTR_192 -#ifndef NO_AES_192 - vldm.32 r12!, {q11-q13} - mov lr, #1 - cmp r4, #1 - blt L_aes_ctr_encrypt_arm32_crypto_192_done - beq L_aes_ctr_encrypt_arm32_crypto_192_start_1 - adds r8, r8, #1 - adcs r7, r7, #0 - adcs r6, r6, #0 - adc r5, r5, #0 - vmov d3, r7, r8 - vmov d2, r5, r6 - vrev32.8 q1, q1 -L_aes_ctr_encrypt_arm32_crypto_192_start_2: - aese.8 q0, q3 - aesmc.8 q0, q0 - aese.8 q1, q3 - aesmc.8 q1, q1 - adds r8, r8, #1 - aese.8 q0, q4 - aesmc.8 q0, q0 - aese.8 q1, q4 - aesmc.8 q1, q1 - adcs r7, r7, #0 - aese.8 q0, q5 - aesmc.8 q0, q0 - aese.8 q1, q5 - aesmc.8 q1, q1 - adcs r6, r6, #0 - aese.8 q0, q6 - aesmc.8 q0, q0 - aese.8 q1, q6 - aesmc.8 q1, q1 - adc r5, r5, #0 - aese.8 q0, q7 - aesmc.8 q0, q0 - aese.8 q1, q7 - aesmc.8 q1, q1 - aese.8 q0, q8 - aesmc.8 q0, q0 - aese.8 q1, q8 - aesmc.8 q1, q1 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q1, q9 - aesmc.8 q1, q1 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q1, q10 - aesmc.8 q1, q1 - aese.8 q0, q11 - aesmc.8 q0, q0 - aese.8 q1, q11 - aesmc.8 q1, q1 - aese.8 q0, q12 - aesmc.8 q0, q0 - aese.8 q1, q12 - aesmc.8 q1, q1 - vld1.32 {q14}, [r12]! - vmov d5, r7, r8 - aese.8 q0, q13 - aesmc.8 q0, q0 - aese.8 q1, q13 - aesmc.8 q1, q1 - vld1.32 {q15}, [r12] - vmov d4, r5, r6 - aese.8 q0, q14 - aese.8 q1, q14 - sub r4, r4, #2 - veor.32 q0, q0, q15 - veor.32 q1, q1, q15 - adds r8, r8, #1 - vld1.8 {q14-q15}, [r0]! - adcs r7, r7, #0 - sub r12, r12, #16 - veor.32 q14, q14, q0 - veor.32 q15, q15, q1 - adcs r6, r6, #0 - vrev32.8 q0, q2 - adc r5, r5, #0 - vmov d2, r5, r6 - vmov d3, r7, r8 - cmp r4, #1 - vst1.8 {q14-q15}, [r1]! - vrev32.8 q1, q1 - bgt L_aes_ctr_encrypt_arm32_crypto_192_start_2 - mov lr, #0 - blt L_aes_ctr_encrypt_arm32_crypto_192_done -L_aes_ctr_encrypt_arm32_crypto_192_start_1: - aese.8 q0, q3 - aesmc.8 q0, q0 - adds r8, r8, lr - aese.8 q0, q4 - aesmc.8 q0, q0 - adcs r7, r7, #0 - aese.8 q0, q5 - aesmc.8 q0, q0 - adcs r6, r6, #0 - aese.8 q0, q6 - aesmc.8 q0, q0 - adc r5, r5, #0 - aese.8 q0, q7 - aesmc.8 q0, q0 - vmov d5, r7, r8 - aese.8 q0, q8 - aesmc.8 q0, q0 - vmov d4, r5, r6 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q0, q11 - aesmc.8 q0, q0 - aese.8 q0, q12 - aesmc.8 q0, q0 - vld1.32 {q14}, [r12]! - aese.8 q0, q13 - aesmc.8 q0, q0 - vld1.32 {q15}, [r12] - aese.8 q0, q14 - vld1.8 {q14}, [r0]! - veor.32 q0, q0, q15 - veor.32 q14, q14, q0 - sub r12, r12, #16 - vst1.8 {q14}, [r1]! - mov lr, #1 - vrev32.8 q0, q2 -L_aes_ctr_encrypt_arm32_crypto_192_done: - cmp r2, #0 - beq L_aes_ctr_encrypt_arm32_crypto_192_partial_done - ldr r4, [sp, #96] - aese.8 q0, q3 - aesmc.8 q0, q0 - adds r8, r8, lr - aese.8 q0, q4 - aesmc.8 q0, q0 - adcs r7, r7, #0 - aese.8 q0, q5 - aesmc.8 q0, q0 - adcs r6, r6, #0 - aese.8 q0, q6 - aesmc.8 q0, q0 - adc r5, r5, #0 - aese.8 q0, q7 - aesmc.8 q0, q0 - vmov d5, r7, r8 - aese.8 q0, q8 - aesmc.8 q0, q0 - vmov d4, r5, r6 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q0, q11 - aesmc.8 q0, q0 - aese.8 q0, q12 - aesmc.8 q0, q0 - vld1.32 {q14}, [r12]! - aese.8 q0, q13 - aesmc.8 q0, q0 - vld1.32 {q15}, [r12] - ldr lr, [sp, #92] - aese.8 q0, q14 - veor.32 q0, q0, q15 - vst1.32 {q0}, [lr] - vmov q0, q2 - mov r5, #16 - sub r5, r5, r2 -L_aes_ctr_encrypt_arm32_crypto_192_start_byte: - ldrb r7, [lr], #1 - ldrb r8, [r0], #1 - eor r7, r7, r8 - subs r2, r2, #1 - strb r7, [r1], #1 - bgt L_aes_ctr_encrypt_arm32_crypto_192_start_byte - vrev32.8 q0, q2 - str r5, [r4] -L_aes_ctr_encrypt_arm32_crypto_192_partial_done: -#endif /* !NO_AES_192 */ - b L_aes_ctr_encrypt_arm32_crypto_done - # AES_CTR_256 -L_aes_ctr_encrypt_arm32_crypto_start_256: -#ifndef NO_AES_256 - vldm.32 r12!, {q11-q13} - mov lr, #1 - cmp r4, #1 - blt L_aes_ctr_encrypt_arm32_crypto_256_done - beq L_aes_ctr_encrypt_arm32_crypto_256_start_1 - adds r8, r8, #1 - adcs r7, r7, #0 - adcs r6, r6, #0 - adc r5, r5, #0 - vmov d3, r7, r8 - vmov d2, r5, r6 - vrev32.8 q1, q1 -L_aes_ctr_encrypt_arm32_crypto_256_start_2: - aese.8 q0, q3 - aesmc.8 q0, q0 - aese.8 q1, q3 - aesmc.8 q1, q1 - adds r8, r8, #1 - aese.8 q0, q4 - aesmc.8 q0, q0 - aese.8 q1, q4 - aesmc.8 q1, q1 - adcs r7, r7, #0 - aese.8 q0, q5 - aesmc.8 q0, q0 - aese.8 q1, q5 - aesmc.8 q1, q1 - adcs r6, r6, #0 - aese.8 q0, q6 - aesmc.8 q0, q0 - aese.8 q1, q6 - aesmc.8 q1, q1 - adc r5, r5, #0 - aese.8 q0, q7 - aesmc.8 q0, q0 - aese.8 q1, q7 - aesmc.8 q1, q1 - aese.8 q0, q8 - aesmc.8 q0, q0 - aese.8 q1, q8 - aesmc.8 q1, q1 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q1, q9 - aesmc.8 q1, q1 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q1, q10 - aesmc.8 q1, q1 - aese.8 q0, q11 - aesmc.8 q0, q0 - aese.8 q1, q11 - aesmc.8 q1, q1 - aese.8 q0, q12 - aesmc.8 q0, q0 - aese.8 q1, q12 - aesmc.8 q1, q1 - vld1.32 {q14}, [r12]! - vmov d5, r7, r8 - aese.8 q0, q13 - aesmc.8 q0, q0 - aese.8 q1, q13 - aesmc.8 q1, q1 - vld1.32 {q15}, [r12]! - vmov d4, r5, r6 - aese.8 q0, q14 - aesmc.8 q0, q0 - aese.8 q1, q14 - aesmc.8 q1, q1 - vld1.32 {q14}, [r12]! - adds r8, r8, #1 - aese.8 q0, q15 - aesmc.8 q0, q0 - aese.8 q1, q15 - aesmc.8 q1, q1 - vld1.32 {q15}, [r12] - adcs r7, r7, #0 - aese.8 q0, q14 - aese.8 q1, q14 - sub r4, r4, #2 - veor.32 q0, q0, q15 - veor.32 q1, q1, q15 - adcs r6, r6, #0 - vld1.8 {q14-q15}, [r0]! - sub r12, r12, #48 - veor.32 q14, q14, q0 - veor.32 q15, q15, q1 - adc r5, r5, #0 - vrev32.8 q0, q2 - vmov d2, r5, r6 - vmov d3, r7, r8 - cmp r4, #1 - vst1.8 {q14-q15}, [r1]! - vrev32.8 q1, q1 - bgt L_aes_ctr_encrypt_arm32_crypto_256_start_2 - mov lr, #0 - blt L_aes_ctr_encrypt_arm32_crypto_256_done -L_aes_ctr_encrypt_arm32_crypto_256_start_1: - aese.8 q0, q3 - aesmc.8 q0, q0 - adds r8, r8, lr - aese.8 q0, q4 - aesmc.8 q0, q0 - adcs r7, r7, #0 - aese.8 q0, q5 - aesmc.8 q0, q0 - adcs r6, r6, #0 - aese.8 q0, q6 - aesmc.8 q0, q0 - adc r5, r5, #0 - aese.8 q0, q7 - aesmc.8 q0, q0 - vmov d5, r7, r8 - aese.8 q0, q8 - aesmc.8 q0, q0 - vmov d4, r5, r6 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q0, q11 - aesmc.8 q0, q0 - aese.8 q0, q12 - aesmc.8 q0, q0 - vld1.32 {q14}, [r12]! - aese.8 q0, q13 - aesmc.8 q0, q0 - vld1.32 {q15}, [r12]! - aese.8 q0, q14 - aesmc.8 q0, q0 - vld1.32 {q14}, [r12]! - aese.8 q0, q15 - aesmc.8 q0, q0 - vld1.32 {q15}, [r12] - aese.8 q0, q14 - vld1.8 {q14}, [r0]! - veor.32 q0, q0, q15 - veor.32 q14, q14, q0 - sub r12, r12, #48 - vst1.8 {q14}, [r1]! - mov lr, #1 - vrev32.8 q0, q2 -L_aes_ctr_encrypt_arm32_crypto_256_done: - cmp r2, #0 - beq L_aes_ctr_encrypt_arm32_crypto_256_partial_done - ldr r4, [sp, #96] - aese.8 q0, q3 - aesmc.8 q0, q0 - adds r8, r8, lr - aese.8 q0, q4 - aesmc.8 q0, q0 - adcs r7, r7, #0 - aese.8 q0, q5 - aesmc.8 q0, q0 - adcs r6, r6, #0 - aese.8 q0, q6 - aesmc.8 q0, q0 - adc r5, r5, #0 - aese.8 q0, q7 - aesmc.8 q0, q0 - vmov d5, r7, r8 - aese.8 q0, q8 - aesmc.8 q0, q0 - vmov d4, r5, r6 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q0, q11 - aesmc.8 q0, q0 - aese.8 q0, q12 - aesmc.8 q0, q0 - vld1.32 {q14}, [r12]! - aese.8 q0, q13 - aesmc.8 q0, q0 - vld1.32 {q15}, [r12]! - aese.8 q0, q14 - aesmc.8 q0, q0 - vld1.32 {q14}, [r12]! - aese.8 q0, q15 - aesmc.8 q0, q0 - vld1.32 {q15}, [r12] - ldr lr, [sp, #92] - aese.8 q0, q14 - veor.32 q0, q0, q15 - vst1.32 {q0}, [lr] - vmov q0, q2 - mov r5, #16 - sub r5, r5, r2 -L_aes_ctr_encrypt_arm32_crypto_256_start_byte: - ldrb r7, [lr], #1 - ldrb r8, [r0], #1 - eor r7, r7, r8 - subs r2, r2, #1 - strb r7, [r1], #1 - bgt L_aes_ctr_encrypt_arm32_crypto_256_start_byte - vrev32.8 q0, q2 - str r5, [r4] -L_aes_ctr_encrypt_arm32_crypto_256_partial_done: -#endif /* !NO_AES_256 */ - b L_aes_ctr_encrypt_arm32_crypto_done - # AES_CTR_128 -L_aes_ctr_encrypt_arm32_crypto_start_128: -#ifndef NO_AES_128 - vldm.32 r12!, {q11-q13} - mov lr, #1 - cmp r4, #1 - blt L_aes_ctr_encrypt_arm32_crypto_128_done - beq L_aes_ctr_encrypt_arm32_crypto_128_start_1 - adds r8, r8, #1 - adcs r7, r7, #0 - adcs r6, r6, #0 - adc r5, r5, #0 - vmov d3, r7, r8 - vmov d2, r5, r6 - vrev32.8 q1, q1 -L_aes_ctr_encrypt_arm32_crypto_128_start_2: - aese.8 q0, q3 - aesmc.8 q0, q0 - aese.8 q1, q3 - aesmc.8 q1, q1 - adds r8, r8, #1 - aese.8 q0, q4 - aesmc.8 q0, q0 - aese.8 q1, q4 - aesmc.8 q1, q1 - adcs r7, r7, #0 - aese.8 q0, q5 - aesmc.8 q0, q0 - aese.8 q1, q5 - aesmc.8 q1, q1 - adcs r6, r6, #0 - aese.8 q0, q6 - aesmc.8 q0, q0 - aese.8 q1, q6 - aesmc.8 q1, q1 - adc r5, r5, #0 - aese.8 q0, q7 - aesmc.8 q0, q0 - aese.8 q1, q7 - aesmc.8 q1, q1 - vmov d5, r7, r8 - aese.8 q0, q8 - aesmc.8 q0, q0 - aese.8 q1, q8 - aesmc.8 q1, q1 - vmov d4, r5, r6 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q1, q9 - aesmc.8 q1, q1 - adds r8, r8, #1 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q1, q10 - aesmc.8 q1, q1 - adcs r7, r7, #0 - aese.8 q0, q11 - aesmc.8 q0, q0 - aese.8 q1, q11 - aesmc.8 q1, q1 - adcs r6, r6, #0 - vld1.8 {q14-q15}, [r0]! - adc r5, r5, #0 - aese.8 q0, q12 - aese.8 q1, q12 - sub r4, r4, #2 - veor.32 q14, q14, q13 - veor.32 q15, q15, q13 - veor.32 q14, q14, q0 - veor.32 q15, q15, q1 - vrev32.8 q0, q2 - vmov d2, r5, r6 - vmov d3, r7, r8 - cmp r4, #1 - vst1.8 {q14-q15}, [r1]! - vrev32.8 q1, q1 - bgt L_aes_ctr_encrypt_arm32_crypto_128_start_2 - mov lr, #0 - blt L_aes_ctr_encrypt_arm32_crypto_128_done -L_aes_ctr_encrypt_arm32_crypto_128_start_1: - aese.8 q0, q3 - aesmc.8 q0, q0 - adds r8, r8, lr - aese.8 q0, q4 - aesmc.8 q0, q0 - adcs r7, r7, #0 - aese.8 q0, q5 - aesmc.8 q0, q0 - adcs r6, r6, #0 - aese.8 q0, q6 - aesmc.8 q0, q0 - adc r5, r5, #0 - aese.8 q0, q7 - aesmc.8 q0, q0 - vmov d5, r7, r8 - aese.8 q0, q8 - aesmc.8 q0, q0 - vmov d4, r5, r6 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q0, q11 - aesmc.8 q0, q0 - vld1.8 {q14}, [r0]! - aese.8 q0, q12 - veor.32 q0, q0, q13 - veor.32 q14, q14, q0 - vst1.8 {q14}, [r1]! - mov lr, #1 - vrev32.8 q0, q2 -L_aes_ctr_encrypt_arm32_crypto_128_done: - cmp r2, #0 - beq L_aes_ctr_encrypt_arm32_crypto_128_partial_done - ldr r4, [sp, #96] - aese.8 q0, q3 - aesmc.8 q0, q0 - adds r8, r8, lr - aese.8 q0, q4 - aesmc.8 q0, q0 - adcs r7, r7, #0 - aese.8 q0, q5 - aesmc.8 q0, q0 - adcs r6, r6, #0 - aese.8 q0, q6 - aesmc.8 q0, q0 - adc r5, r5, #0 - aese.8 q0, q7 - aesmc.8 q0, q0 - vmov d5, r7, r8 - aese.8 q0, q8 - aesmc.8 q0, q0 - vmov d4, r5, r6 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q0, q11 - aesmc.8 q0, q0 - ldr lr, [sp, #92] - aese.8 q0, q12 - veor.32 q0, q0, q13 - vst1.32 {q0}, [lr] - vmov q0, q2 - mov r5, #16 - sub r5, r5, r2 -L_aes_ctr_encrypt_arm32_crypto_128_start_byte: - ldrb r7, [lr], #1 - ldrb r8, [r0], #1 - eor r7, r7, r8 - subs r2, r2, #1 - strb r7, [r1], #1 - bgt L_aes_ctr_encrypt_arm32_crypto_128_start_byte - vrev32.8 q0, q2 - str r5, [r4] -L_aes_ctr_encrypt_arm32_crypto_128_partial_done: -#endif /* !NO_AES_128 */ -L_aes_ctr_encrypt_arm32_crypto_done: - vst1.32 {q0}, [r3] - vpop {d8-d15} - pop {r4, r5, r6, r7, r8, pc} - .size AES_CTR_encrypt_AARCH32,.-AES_CTR_encrypt_AARCH32 -#endif /* WOLFSSL_AES_COUNTER */ -#ifdef HAVE_AESGCM - .text - .align 4 - .globl AES_GCM_set_key_AARCH32 - .type AES_GCM_set_key_AARCH32, %function -AES_GCM_set_key_AARCH32: - vpush {d8-d9} - vld1.8 {q0}, [r0] - vld1.8 {q1-q2}, [r1]! - vld1.8 {q3-q4}, [r1]! - aese.8 q0, q1 - aesmc.8 q0, q0 - aese.8 q0, q2 - aesmc.8 q0, q0 - aese.8 q0, q3 - aesmc.8 q0, q0 - aese.8 q0, q4 - aesmc.8 q0, q0 - vld1.8 {q1-q2}, [r1]! - vld1.8 {q3-q4}, [r1]! - aese.8 q0, q1 - aesmc.8 q0, q0 - aese.8 q0, q2 - aesmc.8 q0, q0 - aese.8 q0, q3 - aesmc.8 q0, q0 - aese.8 q0, q4 - aesmc.8 q0, q0 - subs r3, r3, #10 - vld1.8 {q1-q2}, [r1]! - aese.8 q0, q1 - aesmc.8 q0, q0 - aese.8 q0, q2 - beq L_aes_gcm_set_key_arm32_crypto_round_done - vld1.8 {q1-q2}, [r1]! - subs r3, r3, #2 - aesmc.8 q0, q0 - aese.8 q0, q1 - aesmc.8 q0, q0 - aese.8 q0, q2 - beq L_aes_gcm_set_key_arm32_crypto_round_done - vld1.8 {q1-q2}, [r1]! - aesmc.8 q0, q0 - aese.8 q0, q1 - aesmc.8 q0, q0 - aese.8 q0, q2 -L_aes_gcm_set_key_arm32_crypto_round_done: - vld1.8 {q1}, [r1] - veor q0, q0, q1 - vmov.i8 q1, #0x55 - vshl.u8 q2, q0, #1 - vshr.u8 q0, q0, #1 - vbif.8 q0, q2, q1 - vmov.i8 q1, #51 - vshl.u8 q2, q0, #2 - vshr.u8 q0, q0, #2 - vbit.8 q2, q0, q1 - vshl.u8 q0, q2, #4 - vsri.u8 q0, q2, #4 - vst1.32 {q0}, [r2] - vpop {d8-d9} - bx lr - .size AES_GCM_set_key_AARCH32,.-AES_GCM_set_key_AARCH32 - .text - .align 4 - .globl AES_GCM_encrypt_AARCH32 - .type AES_GCM_encrypt_AARCH32, %function -AES_GCM_encrypt_AARCH32: - push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - vpush {d8-d15} - # key - ldr r7, [sp, #120] - # tmp - ldr r9, [sp, #128] - # nonceSz - ldr r12, [sp, #100] - cmp r12, #12 - beq L_aes_gcm_encrypt_arm32_crypto_nonce_setup_done - vmov.i8 q13, #0x87 - # gcm_h - ldr r8, [sp, #124] - veor.8 q6, q6, q6 - vshr.u64 q13, q13, #56 - vld1.32 {q8}, [r8] -L_aes_gcm_encrypt_arm32_crypto_nonce_setup_done: - # Load Nonce - cmp r12, #12 - bne L_aes_gcm_encrypt_arm32_crypto_ghash_nonce - ldr r5, [r3] - ldr r8, [r3, #4] - ldr r12, [r3, #8] - vmov.i32 q6, #0x1000000 - vmov.32 s24, r5 - vmov.32 s25, r8 - vmov.32 s26, r12 - mov r5, #1 - b L_aes_gcm_encrypt_arm32_crypto_done_nonce -L_aes_gcm_encrypt_arm32_crypto_ghash_nonce: - lsr r10, r12, #4 - cmp r10, #0 - beq L_aes_gcm_encrypt_arm32_crypto_nonce_done -L_aes_gcm_encrypt_arm32_crypto_nonce_start_1: - vld1.32 {q14}, [r3]! - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q6, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q6, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q6 - # Reduce - vmull.p64 q6, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d13 - veor.8 d1, d1, d12 - vmull.p64 q6, d2, d26 - veor.8 q6, q6, q0 - # Done GHASH - subs r10, r10, #1 - bne L_aes_gcm_encrypt_arm32_crypto_nonce_start_1 -L_aes_gcm_encrypt_arm32_crypto_nonce_done: - ands r11, r12, #15 - beq L_aes_gcm_encrypt_arm32_crypto_nonce_partial_done - veor.8 q0, q0, q0 - mov r12, r11 - vst1.32 {q0}, [r9] - cmp r12, #4 - blt L_aes_gcm_encrypt_arm32_crypto_nonce_start_sw -L_aes_gcm_encrypt_arm32_crypto_nonce_start_dw: - ldr r8, [r3], #4 - sub r12, r12, #4 - str r8, [r9], #4 - cmp r12, #4 - bge L_aes_gcm_encrypt_arm32_crypto_nonce_start_dw -L_aes_gcm_encrypt_arm32_crypto_nonce_start_sw: - cmp r12, #2 - blt L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte - ldrh r8, [r3], #2 - sub r12, r12, #2 - strh r8, [r9], #2 -L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte: - cmp r12, #1 - blt L_aes_gcm_encrypt_arm32_crypto_nonce_end_bytes - ldrb r8, [r3], #1 - subs r12, r12, #1 - strb r8, [r9], #1 - bne L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte -L_aes_gcm_encrypt_arm32_crypto_nonce_end_bytes: - sub r9, r9, r11 - vld1.32 {q14}, [r9] - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q6, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q6, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q6 - # Reduce - vmull.p64 q6, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d13 - veor.8 d1, d1, d12 - vmull.p64 q6, d2, d26 - veor.8 q6, q6, q0 - # Done GHASH -L_aes_gcm_encrypt_arm32_crypto_nonce_partial_done: - veor.8 q0, q0, q0 - # nonceSz - ldr r12, [sp, #100] - lsr r10, r12, #29 - lsl r11, r12, #3 - rbit r10, r10 - rbit r11, r11 - vmov.32 s2, r10 - vmov.32 s3, r11 - veor.8 q6, q6, q0 - vmull.p64 q2, d13, d16 - vmull.p64 q12, d12, d17 - vmull.p64 q0, d12, d16 - vmull.p64 q1, d13, d17 - veor.8 q2, q2, q12 - vmull.p64 q6, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d13 - veor.8 d1, d1, d12 - vmull.p64 q6, d2, d26 - veor.8 q6, q6, q0 - vmov.32 r5, s24 - vmov.32 r8, s25 - vmov.32 r12, s26 - rbit r5, r5 - rbit r8, r8 - rbit r12, r12 - rev r5, r5 - rev r8, r8 - rev r12, r12 - vmov.32 s24, r5 - vmov.32 s25, r8 - vmov.32 s26, r12 - vmov.32 r5, s27 - rbit r5, r5 - rev r5, r5 - vmov.32 s27, r5 - rev r5, r5 -L_aes_gcm_encrypt_arm32_crypto_done_nonce: - vldm.32 r7!, {q0-q3} - vldm.32 r7!, {q7-q13} - # nr - ldr r12, [sp, #136] - lsr r10, r2, #4 - cmp r12, #12 - blt L_aes_gcm_encrypt_arm32_crypto_start_128 - bgt L_aes_gcm_encrypt_arm32_crypto_start_256 - # AES_GCM_192 -#ifndef NO_AES_192 - cmp r10, #1 - blt L_aes_gcm_encrypt_arm32_crypto_192_done - beq L_aes_gcm_encrypt_arm32_crypto_192_start_1 -L_aes_gcm_encrypt_arm32_crypto_192_start_2: - add r8, r5, #1 - vmov.8 q4, q6 - add r5, r5, #2 - vmov.8 q5, q6 - rev r8, r8 - rev r12, r5 - vmov s19, r8 - vmov s23, r12 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q5, q0 - aesmc.8 q5, q5 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q5, q1 - aesmc.8 q5, q5 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q5, q2 - aesmc.8 q5, q5 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q5, q3 - aesmc.8 q5, q5 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q5, q7 - aesmc.8 q5, q5 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q5, q8 - aesmc.8 q5, q5 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q5, q9 - aesmc.8 q5, q5 - subs r10, r10, #2 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q5, q10 - aesmc.8 q5, q5 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q5, q11 - aesmc.8 q5, q5 - aese.8 q4, q12 - aesmc.8 q4, q4 - aese.8 q5, q12 - aesmc.8 q5, q5 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - aese.8 q5, q13 - aesmc.8 q5, q5 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - aese.8 q5, q14 - veor.8 q5, q5, q15 - vld1.8 {q14-q15}, [r0]! - sub r7, r7, #16 - veor.8 q14, q14, q4 - veor.8 q15, q15, q5 - vst1.8 {q14-q15}, [r1]! - cmp r10, #1 - bgt L_aes_gcm_encrypt_arm32_crypto_192_start_2 - blt L_aes_gcm_encrypt_arm32_crypto_192_done -L_aes_gcm_encrypt_arm32_crypto_192_start_1: - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q4, q12 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - vld1.8 {q14}, [r0]! - sub r7, r7, #16 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r1]! -L_aes_gcm_encrypt_arm32_crypto_192_done: - ands r11, r2, #15 - beq L_aes_gcm_encrypt_arm32_crypto_192_partial_done - veor.8 q14, q14, q14 - mov r4, r11 - vst1.32 {q14}, [r9] - cmp r4, #4 - blt L_aes_gcm_encrypt_arm32_crypto_192_start_sw -L_aes_gcm_encrypt_arm32_crypto_192_start_dw: - ldr lr, [r0], #4 - sub r4, r4, #4 - str lr, [r9], #4 - cmp r4, #4 - bge L_aes_gcm_encrypt_arm32_crypto_192_start_dw -L_aes_gcm_encrypt_arm32_crypto_192_start_sw: - cmp r4, #2 - blt L_aes_gcm_encrypt_arm32_crypto_192_start_byte - ldrh lr, [r0], #2 - sub r4, r4, #2 - strh lr, [r9], #2 -L_aes_gcm_encrypt_arm32_crypto_192_start_byte: - cmp r4, #1 - blt L_aes_gcm_encrypt_arm32_crypto_192_end_bytes - ldrb lr, [r0], #1 - subs r4, r4, #1 - strb lr, [r9], #1 - bne L_aes_gcm_encrypt_arm32_crypto_192_start_byte -L_aes_gcm_encrypt_arm32_crypto_192_end_bytes: - sub r9, r9, r11 - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q4, q12 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - vld1.8 {q14}, [r9] - sub r7, r7, #16 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r9] - mov r4, r11 - cmp r4, #4 - blt L_aes_gcm_encrypt_arm32_crypto_192_out_start_sw -L_aes_gcm_encrypt_arm32_crypto_192_out_start_dw: - ldr lr, [r9], #4 - sub r4, r4, #4 - str lr, [r1], #4 - cmp r4, #4 - bge L_aes_gcm_encrypt_arm32_crypto_192_out_start_dw -L_aes_gcm_encrypt_arm32_crypto_192_out_start_sw: - cmp r4, #2 - blt L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte - ldrh lr, [r9], #2 - sub r4, r4, #2 - strh lr, [r1], #2 -L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte: - cmp r4, #1 - blt L_aes_gcm_encrypt_arm32_crypto_192_out_end_bytes - ldrb lr, [r9], #1 - subs r4, r4, #1 - strb lr, [r1], #1 - bne L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte -L_aes_gcm_encrypt_arm32_crypto_192_out_end_bytes: -L_aes_gcm_encrypt_arm32_crypto_192_partial_done: - # Finish - add r8, r2, #15 - sub r8, r5, r8, lsr #4 - rev r8, r8 - vmov.32 s27, r8 - aese.8 q6, q0 - aesmc.8 q6, q6 - aese.8 q6, q1 - aesmc.8 q6, q6 - aese.8 q6, q2 - aesmc.8 q6, q6 - aese.8 q6, q3 - aesmc.8 q6, q6 - aese.8 q6, q7 - aesmc.8 q6, q6 - aese.8 q6, q8 - aesmc.8 q6, q6 - aese.8 q6, q9 - aesmc.8 q6, q6 - aese.8 q6, q10 - aesmc.8 q6, q6 - aese.8 q6, q11 - aesmc.8 q6, q6 - aese.8 q6, q12 - aesmc.8 q6, q6 - vld1.32 {q14}, [r7]! - aese.8 q6, q13 - aesmc.8 q6, q6 - vld1.32 {q15}, [r7] - aese.8 q6, q14 - veor.8 q6, q6, q15 - sub r7, r7, #16 -#endif /* !NO_AES_192 */ - b L_aes_gcm_encrypt_arm32_crypto_done_enc - # AES_GCM_256 -L_aes_gcm_encrypt_arm32_crypto_start_256: -#ifndef NO_AES_256 - cmp r10, #1 - blt L_aes_gcm_encrypt_arm32_crypto_256_done - beq L_aes_gcm_encrypt_arm32_crypto_256_start_1 -L_aes_gcm_encrypt_arm32_crypto_256_start_2: - add r8, r5, #1 - vmov.8 q4, q6 - add r5, r5, #2 - vmov.8 q5, q6 - rev r8, r8 - rev r12, r5 - vmov s19, r8 - vmov s23, r12 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q5, q0 - aesmc.8 q5, q5 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q5, q1 - aesmc.8 q5, q5 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q5, q2 - aesmc.8 q5, q5 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q5, q3 - aesmc.8 q5, q5 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q5, q7 - aesmc.8 q5, q5 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q5, q8 - aesmc.8 q5, q5 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q5, q9 - aesmc.8 q5, q5 - subs r10, r10, #2 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q5, q10 - aesmc.8 q5, q5 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q5, q11 - aesmc.8 q5, q5 - aese.8 q4, q12 - aesmc.8 q4, q4 - aese.8 q5, q12 - aesmc.8 q5, q5 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - aese.8 q5, q13 - aesmc.8 q5, q5 - vld1.32 {q15}, [r7]! - aese.8 q4, q14 - aesmc.8 q4, q4 - aese.8 q5, q14 - aesmc.8 q5, q5 - vld1.32 {q14}, [r7]! - aese.8 q4, q15 - aesmc.8 q4, q4 - aese.8 q5, q15 - aesmc.8 q5, q5 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - aese.8 q5, q14 - veor.8 q5, q5, q15 - vld1.8 {q14-q15}, [r0]! - sub r7, r7, #48 - veor.8 q14, q14, q4 - veor.8 q15, q15, q5 - vst1.8 {q14-q15}, [r1]! - cmp r10, #1 - bgt L_aes_gcm_encrypt_arm32_crypto_256_start_2 - blt L_aes_gcm_encrypt_arm32_crypto_256_done -L_aes_gcm_encrypt_arm32_crypto_256_start_1: - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q4, q12 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7]! - aese.8 q4, q14 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q15 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - vld1.8 {q14}, [r0]! - sub r7, r7, #48 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r1]! -L_aes_gcm_encrypt_arm32_crypto_256_done: - ands r11, r2, #15 - beq L_aes_gcm_encrypt_arm32_crypto_256_partial_done - veor.8 q14, q14, q14 - mov r4, r11 - vst1.32 {q14}, [r9] - cmp r4, #4 - blt L_aes_gcm_encrypt_arm32_crypto_256_start_sw -L_aes_gcm_encrypt_arm32_crypto_256_start_dw: - ldr lr, [r0], #4 - sub r4, r4, #4 - str lr, [r9], #4 - cmp r4, #4 - bge L_aes_gcm_encrypt_arm32_crypto_256_start_dw -L_aes_gcm_encrypt_arm32_crypto_256_start_sw: - cmp r4, #2 - blt L_aes_gcm_encrypt_arm32_crypto_256_start_byte - ldrh lr, [r0], #2 - sub r4, r4, #2 - strh lr, [r9], #2 -L_aes_gcm_encrypt_arm32_crypto_256_start_byte: - cmp r4, #1 - blt L_aes_gcm_encrypt_arm32_crypto_256_end_bytes - ldrb lr, [r0], #1 - subs r4, r4, #1 - strb lr, [r9], #1 - bne L_aes_gcm_encrypt_arm32_crypto_256_start_byte -L_aes_gcm_encrypt_arm32_crypto_256_end_bytes: - sub r9, r9, r11 - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q4, q12 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7]! - aese.8 q4, q14 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q15 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - vld1.8 {q14}, [r9] - sub r7, r7, #48 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r9] - mov r4, r11 - cmp r4, #4 - blt L_aes_gcm_encrypt_arm32_crypto_256_out_start_sw -L_aes_gcm_encrypt_arm32_crypto_256_out_start_dw: - ldr lr, [r9], #4 - sub r4, r4, #4 - str lr, [r1], #4 - cmp r4, #4 - bge L_aes_gcm_encrypt_arm32_crypto_256_out_start_dw -L_aes_gcm_encrypt_arm32_crypto_256_out_start_sw: - cmp r4, #2 - blt L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte - ldrh lr, [r9], #2 - sub r4, r4, #2 - strh lr, [r1], #2 -L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte: - cmp r4, #1 - blt L_aes_gcm_encrypt_arm32_crypto_256_out_end_bytes - ldrb lr, [r9], #1 - subs r4, r4, #1 - strb lr, [r1], #1 - bne L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte -L_aes_gcm_encrypt_arm32_crypto_256_out_end_bytes: -L_aes_gcm_encrypt_arm32_crypto_256_partial_done: - # Finish - add r8, r2, #15 - sub r8, r5, r8, lsr #4 - rev r8, r8 - vmov.32 s27, r8 - aese.8 q6, q0 - aesmc.8 q6, q6 - aese.8 q6, q1 - aesmc.8 q6, q6 - aese.8 q6, q2 - aesmc.8 q6, q6 - aese.8 q6, q3 - aesmc.8 q6, q6 - aese.8 q6, q7 - aesmc.8 q6, q6 - aese.8 q6, q8 - aesmc.8 q6, q6 - aese.8 q6, q9 - aesmc.8 q6, q6 - aese.8 q6, q10 - aesmc.8 q6, q6 - aese.8 q6, q11 - aesmc.8 q6, q6 - aese.8 q6, q12 - aesmc.8 q6, q6 - vld1.32 {q14}, [r7]! - aese.8 q6, q13 - aesmc.8 q6, q6 - vld1.32 {q15}, [r7]! - aese.8 q6, q14 - aesmc.8 q6, q6 - vld1.32 {q14}, [r7]! - aese.8 q6, q15 - aesmc.8 q6, q6 - vld1.32 {q15}, [r7] - aese.8 q6, q14 - veor.8 q6, q6, q15 - sub r7, r7, #48 -#endif /* !NO_AES_256 */ - b L_aes_gcm_encrypt_arm32_crypto_done_enc - # AES_GCM_128 -L_aes_gcm_encrypt_arm32_crypto_start_128: -#ifndef NO_AES_128 - cmp r10, #1 - blt L_aes_gcm_encrypt_arm32_crypto_128_done - beq L_aes_gcm_encrypt_arm32_crypto_128_start_1 -L_aes_gcm_encrypt_arm32_crypto_128_start_2: - add r8, r5, #1 - vmov.8 q4, q6 - add r5, r5, #2 - vmov.8 q5, q6 - rev r8, r8 - rev r12, r5 - vmov s19, r8 - vmov s23, r12 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q5, q0 - aesmc.8 q5, q5 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q5, q1 - aesmc.8 q5, q5 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q5, q2 - aesmc.8 q5, q5 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q5, q3 - aesmc.8 q5, q5 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q5, q7 - aesmc.8 q5, q5 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q5, q8 - aesmc.8 q5, q5 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q5, q9 - aesmc.8 q5, q5 - subs r10, r10, #2 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q5, q10 - aesmc.8 q5, q5 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q5, q11 - aesmc.8 q5, q5 - vld1.8 {q14-q15}, [r0]! - aese.8 q4, q12 - veor.8 q4, q4, q13 - aese.8 q5, q12 - veor.8 q5, q5, q13 - veor.8 q14, q14, q4 - veor.8 q15, q15, q5 - vst1.8 {q14-q15}, [r1]! - cmp r10, #1 - bgt L_aes_gcm_encrypt_arm32_crypto_128_start_2 - blt L_aes_gcm_encrypt_arm32_crypto_128_done -L_aes_gcm_encrypt_arm32_crypto_128_start_1: - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - vld1.8 {q14}, [r0]! - aese.8 q4, q12 - veor.8 q4, q4, q13 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r1]! -L_aes_gcm_encrypt_arm32_crypto_128_done: - ands r11, r2, #15 - beq L_aes_gcm_encrypt_arm32_crypto_128_partial_done - veor.8 q14, q14, q14 - mov r4, r11 - vst1.32 {q14}, [r9] - cmp r4, #4 - blt L_aes_gcm_encrypt_arm32_crypto_128_start_sw -L_aes_gcm_encrypt_arm32_crypto_128_start_dw: - ldr lr, [r0], #4 - sub r4, r4, #4 - str lr, [r9], #4 - cmp r4, #4 - bge L_aes_gcm_encrypt_arm32_crypto_128_start_dw -L_aes_gcm_encrypt_arm32_crypto_128_start_sw: - cmp r4, #2 - blt L_aes_gcm_encrypt_arm32_crypto_128_start_byte - ldrh lr, [r0], #2 - sub r4, r4, #2 - strh lr, [r9], #2 -L_aes_gcm_encrypt_arm32_crypto_128_start_byte: - cmp r4, #1 - blt L_aes_gcm_encrypt_arm32_crypto_128_end_bytes - ldrb lr, [r0], #1 - subs r4, r4, #1 - strb lr, [r9], #1 - bne L_aes_gcm_encrypt_arm32_crypto_128_start_byte -L_aes_gcm_encrypt_arm32_crypto_128_end_bytes: - sub r9, r9, r11 - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - vld1.8 {q14}, [r9] - aese.8 q4, q12 - veor.8 q4, q4, q13 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r9] - mov r4, r11 - cmp r4, #4 - blt L_aes_gcm_encrypt_arm32_crypto_128_out_start_sw -L_aes_gcm_encrypt_arm32_crypto_128_out_start_dw: - ldr lr, [r9], #4 - sub r4, r4, #4 - str lr, [r1], #4 - cmp r4, #4 - bge L_aes_gcm_encrypt_arm32_crypto_128_out_start_dw -L_aes_gcm_encrypt_arm32_crypto_128_out_start_sw: - cmp r4, #2 - blt L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte - ldrh lr, [r9], #2 - sub r4, r4, #2 - strh lr, [r1], #2 -L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte: - cmp r4, #1 - blt L_aes_gcm_encrypt_arm32_crypto_128_out_end_bytes - ldrb lr, [r9], #1 - subs r4, r4, #1 - strb lr, [r1], #1 - bne L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte -L_aes_gcm_encrypt_arm32_crypto_128_out_end_bytes: -L_aes_gcm_encrypt_arm32_crypto_128_partial_done: - # Finish - add r8, r2, #15 - sub r8, r5, r8, lsr #4 - rev r8, r8 - vmov.32 s27, r8 - aese.8 q6, q0 - aesmc.8 q6, q6 - aese.8 q6, q1 - aesmc.8 q6, q6 - aese.8 q6, q2 - aesmc.8 q6, q6 - aese.8 q6, q3 - aesmc.8 q6, q6 - aese.8 q6, q7 - aesmc.8 q6, q6 - aese.8 q6, q8 - aesmc.8 q6, q6 - aese.8 q6, q9 - aesmc.8 q6, q6 - aese.8 q6, q10 - aesmc.8 q6, q6 - aese.8 q6, q11 - aesmc.8 q6, q6 - aese.8 q6, q12 - veor.8 q6, q6, q13 -#endif /* !NO_AES_128 */ -L_aes_gcm_encrypt_arm32_crypto_done_enc: - # aadSz - ldr r6, [sp, #116] - # gcm_h - ldr r8, [sp, #124] - sub r1, r1, r2 - vmov.i8 q13, #0x87 - veor.8 q7, q7, q7 - vshr.u64 q13, q13, #56 - vld1.32 {q8}, [r8] - orr r10, r6, r2 - cmp r10, #32 - blt L_aes_gcm_encrypt_arm32_crypto_h_done - # Square H => H^2 - vmull.p64 q1, d17, d17 - vmull.p64 q0, d16, d16 - vmull.p64 q9, d3, d27 - veor.8 d2, d2, d19 - veor.8 d1, d1, d18 - vmull.p64 q9, d2, d26 - veor.8 q9, q9, q0 - cmp r10, #0x40 - blt L_aes_gcm_encrypt_arm32_crypto_h_done - # Multiply H and H^2 => H^3 - vmull.p64 q2, d17, d18 - vmull.p64 q3, d16, d19 - vmull.p64 q0, d16, d18 - vmull.p64 q1, d17, d19 - veor.8 q2, q2, q3 - # Reduce - vmull.p64 q10, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d21 - veor.8 d1, d1, d20 - vmull.p64 q10, d2, d26 - veor.8 q10, q10, q0 - # Square H^2 => H^4 - vmull.p64 q1, d19, d19 - vmull.p64 q0, d18, d18 - vmull.p64 q11, d3, d27 - veor.8 d2, d2, d23 - veor.8 d1, d1, d22 - vmull.p64 q11, d2, d26 - veor.8 q11, q11, q0 - # Done -L_aes_gcm_encrypt_arm32_crypto_h_done: - # aad - ldr r5, [sp, #112] - lsr r10, r6, #4 - cmp r10, #1 - blt L_aes_gcm_encrypt_arm32_crypto_aad_done - beq L_aes_gcm_encrypt_arm32_crypto_aad_start_1 - cmp r10, #4 - blt L_aes_gcm_encrypt_arm32_crypto_aad_start_2 -L_aes_gcm_encrypt_arm32_crypto_aad_start_4: - vldm r5!, {q0-q2} - vmov.i8 q12, #0x55 - vmov.i8 q5, #51 - vshl.u8 q14, q0, #1 - vshl.u8 q15, q1, #1 - vshl.u8 q4, q2, #1 - ldr lr, [r5], #4 - vshr.u8 q0, q0, #1 - vshr.u8 q1, q1, #1 - vshr.u8 q2, q2, #1 - ldr r4, [r5], #4 - vbit.8 q14, q0, q12 - vbit.8 q15, q1, q12 - vbit.8 q4, q2, q12 - ldr r8, [r5], #4 - vshl.u8 q0, q14, #2 - vshl.u8 q1, q15, #2 - vshl.u8 q2, q4, #2 - ldr r12, [r5], #4 - vshr.u8 q14, q14, #2 - vshr.u8 q15, q15, #2 - vshr.u8 q4, q4, #2 - rbit lr, lr - vbit.8 q0, q14, q5 - rbit r4, r4 - vbit.8 q1, q15, q5 - rbit r8, r8 - vbit.8 q2, q4, q5 - rbit r12, r12 - vshl.u8 q14, q0, #4 - rev lr, lr - vshl.u8 q15, q1, #4 - rev r4, r4 - vshl.u8 q4, q2, #4 - rev r8, r8 - vsri.u8 q14, q0, #4 - rev r12, r12 - vsri.u8 q15, q1, #4 - vsri.u8 q4, q2, #4 - vmov d10, lr, r4 - vmov d11, r8, r12 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d11, d16 - vmull.p64 q7, d10, d17 - vmull.p64 q0, d10, d16 - vmull.p64 q1, d11, d17 - veor.8 q2, q2, q7 - # X += C * H^2 - vmull.p64 q3, d8, d18 - vmull.p64 q7, d9, d19 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d9, d18 - vmull.p64 q7, d8, d19 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # X += C * H^3 - vmull.p64 q3, d30, d20 - vmull.p64 q7, d31, d21 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d31, d20 - vmull.p64 q7, d30, d21 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # X += C * H^4 - vmull.p64 q3, d24, d22 - vmull.p64 q7, d25, d23 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d25, d22 - vmull.p64 q7, d24, d23 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH - sub r10, r10, #4 - cmp r10, #4 - bge L_aes_gcm_encrypt_arm32_crypto_aad_start_4 - cmp r10, #1 - blt L_aes_gcm_encrypt_arm32_crypto_aad_done - beq L_aes_gcm_encrypt_arm32_crypto_aad_start_1 -L_aes_gcm_encrypt_arm32_crypto_aad_start_2: - vld1.32 {q14-q15}, [r5]! - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshl.u8 q1, q15, #1 - vshr.u8 q14, q14, #1 - vshr.u8 q15, q15, #1 - vbif.8 q14, q0, q12 - vbif.8 q15, q1, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshl.u8 q1, q15, #2 - vshr.u8 q14, q14, #2 - vshr.u8 q15, q15, #2 - vbit.8 q0, q14, q12 - vbit.8 q1, q15, q12 - vshl.u8 q14, q0, #4 - vshl.u8 q15, q1, #4 - vsri.u8 q14, q0, #4 - vsri.u8 q15, q1, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d31, d16 - vmull.p64 q7, d30, d17 - vmull.p64 q0, d30, d16 - vmull.p64 q1, d31, d17 - veor.8 q2, q2, q7 - # X += C * H^2 - vmull.p64 q3, d24, d18 - vmull.p64 q7, d25, d19 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d25, d18 - vmull.p64 q7, d24, d19 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH - sub r10, r10, #2 - cmp r10, #0 - beq L_aes_gcm_encrypt_arm32_crypto_aad_done -L_aes_gcm_encrypt_arm32_crypto_aad_start_1: - vld1.32 {q14}, [r5]! - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q7, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH -L_aes_gcm_encrypt_arm32_crypto_aad_done: - ands r11, r6, #15 - beq L_aes_gcm_encrypt_arm32_crypto_aad_partial_done - veor.8 q0, q0, q0 - mov r12, r11 - vst1.32 {q0}, [r9] - cmp r12, #4 - blt L_aes_gcm_encrypt_arm32_crypto_aad_start_sw -L_aes_gcm_encrypt_arm32_crypto_aad_start_dw: - ldr r8, [r5], #4 - sub r12, r12, #4 - str r8, [r9], #4 - cmp r12, #4 - bge L_aes_gcm_encrypt_arm32_crypto_aad_start_dw -L_aes_gcm_encrypt_arm32_crypto_aad_start_sw: - cmp r12, #2 - blt L_aes_gcm_encrypt_arm32_crypto_aad_start_byte - ldrh r8, [r5], #2 - sub r12, r12, #2 - strh r8, [r9], #2 -L_aes_gcm_encrypt_arm32_crypto_aad_start_byte: - cmp r12, #1 - blt L_aes_gcm_encrypt_arm32_crypto_aad_end_bytes - ldrb r8, [r5], #1 - subs r12, r12, #1 - strb r8, [r9], #1 - bne L_aes_gcm_encrypt_arm32_crypto_aad_start_byte -L_aes_gcm_encrypt_arm32_crypto_aad_end_bytes: - sub r9, r9, r11 - vld1.32 {q14}, [r9] - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q7, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH -L_aes_gcm_encrypt_arm32_crypto_aad_partial_done: - # out - lsr r10, r2, #4 - cmp r10, #1 - blt L_aes_gcm_encrypt_arm32_crypto_out_done - beq L_aes_gcm_encrypt_arm32_crypto_out_start_1 - cmp r10, #4 - blt L_aes_gcm_encrypt_arm32_crypto_out_start_2 -L_aes_gcm_encrypt_arm32_crypto_out_start_4: - vldm r1!, {q0-q2} - vmov.i8 q12, #0x55 - vmov.i8 q5, #51 - vshl.u8 q14, q0, #1 - vshl.u8 q15, q1, #1 - vshl.u8 q4, q2, #1 - ldr lr, [r1], #4 - vshr.u8 q0, q0, #1 - vshr.u8 q1, q1, #1 - vshr.u8 q2, q2, #1 - ldr r4, [r1], #4 - vbit.8 q14, q0, q12 - vbit.8 q15, q1, q12 - vbit.8 q4, q2, q12 - ldr r8, [r1], #4 - vshl.u8 q0, q14, #2 - vshl.u8 q1, q15, #2 - vshl.u8 q2, q4, #2 - ldr r12, [r1], #4 - vshr.u8 q14, q14, #2 - vshr.u8 q15, q15, #2 - vshr.u8 q4, q4, #2 - rbit lr, lr - vbit.8 q0, q14, q5 - rbit r4, r4 - vbit.8 q1, q15, q5 - rbit r8, r8 - vbit.8 q2, q4, q5 - rbit r12, r12 - vshl.u8 q14, q0, #4 - rev lr, lr - vshl.u8 q15, q1, #4 - rev r4, r4 - vshl.u8 q4, q2, #4 - rev r8, r8 - vsri.u8 q14, q0, #4 - rev r12, r12 - vsri.u8 q15, q1, #4 - vsri.u8 q4, q2, #4 - vmov d10, lr, r4 - vmov d11, r8, r12 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d11, d16 - vmull.p64 q7, d10, d17 - vmull.p64 q0, d10, d16 - vmull.p64 q1, d11, d17 - veor.8 q2, q2, q7 - # X += C * H^2 - vmull.p64 q3, d8, d18 - vmull.p64 q7, d9, d19 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d9, d18 - vmull.p64 q7, d8, d19 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # X += C * H^3 - vmull.p64 q3, d30, d20 - vmull.p64 q7, d31, d21 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d31, d20 - vmull.p64 q7, d30, d21 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # X += C * H^4 - vmull.p64 q3, d24, d22 - vmull.p64 q7, d25, d23 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d25, d22 - vmull.p64 q7, d24, d23 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH - sub r10, r10, #4 - cmp r10, #4 - bge L_aes_gcm_encrypt_arm32_crypto_out_start_4 - cmp r10, #1 - blt L_aes_gcm_encrypt_arm32_crypto_out_done - beq L_aes_gcm_encrypt_arm32_crypto_out_start_1 -L_aes_gcm_encrypt_arm32_crypto_out_start_2: - vld1.32 {q14-q15}, [r1]! - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshl.u8 q1, q15, #1 - vshr.u8 q14, q14, #1 - vshr.u8 q15, q15, #1 - vbif.8 q14, q0, q12 - vbif.8 q15, q1, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshl.u8 q1, q15, #2 - vshr.u8 q14, q14, #2 - vshr.u8 q15, q15, #2 - vbit.8 q0, q14, q12 - vbit.8 q1, q15, q12 - vshl.u8 q14, q0, #4 - vshl.u8 q15, q1, #4 - vsri.u8 q14, q0, #4 - vsri.u8 q15, q1, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d31, d16 - vmull.p64 q7, d30, d17 - vmull.p64 q0, d30, d16 - vmull.p64 q1, d31, d17 - veor.8 q2, q2, q7 - # X += C * H^2 - vmull.p64 q3, d24, d18 - vmull.p64 q7, d25, d19 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d25, d18 - vmull.p64 q7, d24, d19 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH - sub r10, r10, #2 - cmp r10, #0 - beq L_aes_gcm_encrypt_arm32_crypto_out_done -L_aes_gcm_encrypt_arm32_crypto_out_start_1: - vld1.32 {q14}, [r1]! - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q7, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH -L_aes_gcm_encrypt_arm32_crypto_out_done: - ands r11, r2, #15 - beq L_aes_gcm_encrypt_arm32_crypto_out_partial_done - veor.8 q0, q0, q0 - mov r12, r11 - vst1.32 {q0}, [r9] - cmp r12, #4 - blt L_aes_gcm_encrypt_arm32_crypto_out_start_sw -L_aes_gcm_encrypt_arm32_crypto_out_start_dw: - ldr r8, [r1], #4 - sub r12, r12, #4 - str r8, [r9], #4 - cmp r12, #4 - bge L_aes_gcm_encrypt_arm32_crypto_out_start_dw -L_aes_gcm_encrypt_arm32_crypto_out_start_sw: - cmp r12, #2 - blt L_aes_gcm_encrypt_arm32_crypto_out_start_byte - ldrh r8, [r1], #2 - sub r12, r12, #2 - strh r8, [r9], #2 -L_aes_gcm_encrypt_arm32_crypto_out_start_byte: - cmp r12, #1 - blt L_aes_gcm_encrypt_arm32_crypto_out_end_bytes - ldrb r8, [r1], #1 - subs r12, r12, #1 - strb r8, [r9], #1 - bne L_aes_gcm_encrypt_arm32_crypto_out_start_byte -L_aes_gcm_encrypt_arm32_crypto_out_end_bytes: - sub r9, r9, r11 - vld1.32 {q14}, [r9] - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q7, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH -L_aes_gcm_encrypt_arm32_crypto_out_partial_done: - lsr lr, r6, #29 - lsl r6, r6, #3 - rbit lr, lr - rbit r6, r6 - vmov s0, lr - vmov s1, r6 - lsr lr, r2, #29 - lsl r2, r2, #3 - rbit lr, lr - rbit r2, r2 - vmov s2, lr - vmov s3, r2 - veor.8 q7, q7, q0 - vmull.p64 q2, d15, d16 - vmull.p64 q3, d14, d17 - vmull.p64 q0, d14, d16 - vmull.p64 q1, d15, d17 - veor.8 q2, q2, q3 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - vmov.i8 q0, #0x55 - vshl.u8 q1, q7, #1 - vshr.u8 q7, q7, #1 - vbif.8 q7, q1, q0 - vmov.i8 q0, #51 - vshl.u8 q1, q7, #2 - vshr.u8 q7, q7, #2 - vbit.8 q1, q7, q0 - vshl.u8 q7, q1, #4 - vsri.u8 q7, q1, #4 - # DONE - veor.8 q7, q7, q6 - # tag - ldr lr, [sp, #104] - # tagSz - ldr r4, [sp, #108] - cmp r4, #16 - bne L_aes_gcm_encrypt_arm32_crypto_tag_tag_partial - vst1.8 {q7}, [lr] - b L_aes_gcm_encrypt_arm32_crypto_done_gcm -L_aes_gcm_encrypt_arm32_crypto_tag_tag_partial: - vst1.8 {q7}, [r9] - cmp r4, #4 - blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_sw -L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_dw: - ldr r8, [r9], #4 - sub r4, r4, #4 - str r8, [lr], #4 - cmp r4, #4 - bge L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_dw -L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_sw: - cmp r4, #2 - blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte - ldrh r8, [r9], #2 - sub r4, r4, #2 - strh r8, [lr], #2 -L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte: - cmp r4, #1 - blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_end_bytes - ldrb r8, [r9], #1 - subs r4, r4, #1 - strb r8, [lr], #1 - bne L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte -L_aes_gcm_encrypt_arm32_crypto_tag_tag_end_bytes: -L_aes_gcm_encrypt_arm32_crypto_done_gcm: - vpop {d8-d15} - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size AES_GCM_encrypt_AARCH32,.-AES_GCM_encrypt_AARCH32 -#ifdef HAVE_AES_DECRYPT - .text - .align 4 - .globl AES_GCM_decrypt_AARCH32 - .type AES_GCM_decrypt_AARCH32, %function -AES_GCM_decrypt_AARCH32: - push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - vpush {d8-d15} - # key - ldr r7, [sp, #120] - # tmp - ldr r9, [sp, #128] - # aadSz - ldr r6, [sp, #116] - # gcm_h - ldr r8, [sp, #124] - vmov.i8 q13, #0x87 - veor.8 q7, q7, q7 - vshr.u64 q13, q13, #56 - vld1.32 {q8}, [r8] - orr r10, r6, r2 - cmp r10, #32 - blt L_aes_gcm_decrypt_arm32_crypto_h_done - # Square H => H^2 - vmull.p64 q1, d17, d17 - vmull.p64 q0, d16, d16 - vmull.p64 q9, d3, d27 - veor.8 d2, d2, d19 - veor.8 d1, d1, d18 - vmull.p64 q9, d2, d26 - veor.8 q9, q9, q0 - cmp r10, #0x40 - blt L_aes_gcm_decrypt_arm32_crypto_h_done - # Multiply H and H^2 => H^3 - vmull.p64 q2, d17, d18 - vmull.p64 q3, d16, d19 - vmull.p64 q0, d16, d18 - vmull.p64 q1, d17, d19 - veor.8 q2, q2, q3 - # Reduce - vmull.p64 q10, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d21 - veor.8 d1, d1, d20 - vmull.p64 q10, d2, d26 - veor.8 q10, q10, q0 - # Square H^2 => H^4 - vmull.p64 q1, d19, d19 - vmull.p64 q0, d18, d18 - vmull.p64 q11, d3, d27 - veor.8 d2, d2, d23 - veor.8 d1, d1, d22 - vmull.p64 q11, d2, d26 - veor.8 q11, q11, q0 - # Done -L_aes_gcm_decrypt_arm32_crypto_h_done: - # aad - ldr r5, [sp, #112] - lsr r10, r6, #4 - cmp r10, #1 - blt L_aes_gcm_decrypt_arm32_crypto_aad_done - beq L_aes_gcm_decrypt_arm32_crypto_aad_start_1 - cmp r10, #4 - blt L_aes_gcm_decrypt_arm32_crypto_aad_start_2 -L_aes_gcm_decrypt_arm32_crypto_aad_start_4: - vldm r5!, {q0-q2} - vmov.i8 q12, #0x55 - vmov.i8 q5, #51 - vshl.u8 q14, q0, #1 - vshl.u8 q15, q1, #1 - vshl.u8 q4, q2, #1 - ldr lr, [r5], #4 - vshr.u8 q0, q0, #1 - vshr.u8 q1, q1, #1 - vshr.u8 q2, q2, #1 - ldr r4, [r5], #4 - vbit.8 q14, q0, q12 - vbit.8 q15, q1, q12 - vbit.8 q4, q2, q12 - ldr r8, [r5], #4 - vshl.u8 q0, q14, #2 - vshl.u8 q1, q15, #2 - vshl.u8 q2, q4, #2 - ldr r12, [r5], #4 - vshr.u8 q14, q14, #2 - vshr.u8 q15, q15, #2 - vshr.u8 q4, q4, #2 - rbit lr, lr - vbit.8 q0, q14, q5 - rbit r4, r4 - vbit.8 q1, q15, q5 - rbit r8, r8 - vbit.8 q2, q4, q5 - rbit r12, r12 - vshl.u8 q14, q0, #4 - rev lr, lr - vshl.u8 q15, q1, #4 - rev r4, r4 - vshl.u8 q4, q2, #4 - rev r8, r8 - vsri.u8 q14, q0, #4 - rev r12, r12 - vsri.u8 q15, q1, #4 - vsri.u8 q4, q2, #4 - vmov d10, lr, r4 - vmov d11, r8, r12 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d11, d16 - vmull.p64 q7, d10, d17 - vmull.p64 q0, d10, d16 - vmull.p64 q1, d11, d17 - veor.8 q2, q2, q7 - # X += C * H^2 - vmull.p64 q3, d8, d18 - vmull.p64 q7, d9, d19 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d9, d18 - vmull.p64 q7, d8, d19 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # X += C * H^3 - vmull.p64 q3, d30, d20 - vmull.p64 q7, d31, d21 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d31, d20 - vmull.p64 q7, d30, d21 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # X += C * H^4 - vmull.p64 q3, d24, d22 - vmull.p64 q7, d25, d23 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d25, d22 - vmull.p64 q7, d24, d23 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH - sub r10, r10, #4 - cmp r10, #4 - bge L_aes_gcm_decrypt_arm32_crypto_aad_start_4 - cmp r10, #1 - blt L_aes_gcm_decrypt_arm32_crypto_aad_done - beq L_aes_gcm_decrypt_arm32_crypto_aad_start_1 -L_aes_gcm_decrypt_arm32_crypto_aad_start_2: - vld1.32 {q14-q15}, [r5]! - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshl.u8 q1, q15, #1 - vshr.u8 q14, q14, #1 - vshr.u8 q15, q15, #1 - vbif.8 q14, q0, q12 - vbif.8 q15, q1, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshl.u8 q1, q15, #2 - vshr.u8 q14, q14, #2 - vshr.u8 q15, q15, #2 - vbit.8 q0, q14, q12 - vbit.8 q1, q15, q12 - vshl.u8 q14, q0, #4 - vshl.u8 q15, q1, #4 - vsri.u8 q14, q0, #4 - vsri.u8 q15, q1, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d31, d16 - vmull.p64 q7, d30, d17 - vmull.p64 q0, d30, d16 - vmull.p64 q1, d31, d17 - veor.8 q2, q2, q7 - # X += C * H^2 - vmull.p64 q3, d24, d18 - vmull.p64 q7, d25, d19 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d25, d18 - vmull.p64 q7, d24, d19 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH - sub r10, r10, #2 - cmp r10, #0 - beq L_aes_gcm_decrypt_arm32_crypto_aad_done -L_aes_gcm_decrypt_arm32_crypto_aad_start_1: - vld1.32 {q14}, [r5]! - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q7, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH -L_aes_gcm_decrypt_arm32_crypto_aad_done: - ands r11, r6, #15 - beq L_aes_gcm_decrypt_arm32_crypto_aad_partial_done - veor.8 q0, q0, q0 - mov r12, r11 - vst1.32 {q0}, [r9] - cmp r12, #4 - blt L_aes_gcm_decrypt_arm32_crypto_aad_start_sw -L_aes_gcm_decrypt_arm32_crypto_aad_start_dw: - ldr r8, [r5], #4 - sub r12, r12, #4 - str r8, [r9], #4 - cmp r12, #4 - bge L_aes_gcm_decrypt_arm32_crypto_aad_start_dw -L_aes_gcm_decrypt_arm32_crypto_aad_start_sw: - cmp r12, #2 - blt L_aes_gcm_decrypt_arm32_crypto_aad_start_byte - ldrh r8, [r5], #2 - sub r12, r12, #2 - strh r8, [r9], #2 -L_aes_gcm_decrypt_arm32_crypto_aad_start_byte: - cmp r12, #1 - blt L_aes_gcm_decrypt_arm32_crypto_aad_end_bytes - ldrb r8, [r5], #1 - subs r12, r12, #1 - strb r8, [r9], #1 - bne L_aes_gcm_decrypt_arm32_crypto_aad_start_byte -L_aes_gcm_decrypt_arm32_crypto_aad_end_bytes: - sub r9, r9, r11 - vld1.32 {q14}, [r9] - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q7, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH -L_aes_gcm_decrypt_arm32_crypto_aad_partial_done: - # in - lsr r10, r2, #4 - cmp r10, #1 - blt L_aes_gcm_decrypt_arm32_crypto_in_done - beq L_aes_gcm_decrypt_arm32_crypto_in_start_1 - cmp r10, #4 - blt L_aes_gcm_decrypt_arm32_crypto_in_start_2 -L_aes_gcm_decrypt_arm32_crypto_in_start_4: - vldm r0!, {q0-q2} - vmov.i8 q12, #0x55 - vmov.i8 q5, #51 - vshl.u8 q14, q0, #1 - vshl.u8 q15, q1, #1 - vshl.u8 q4, q2, #1 - ldr lr, [r0], #4 - vshr.u8 q0, q0, #1 - vshr.u8 q1, q1, #1 - vshr.u8 q2, q2, #1 - ldr r4, [r0], #4 - vbit.8 q14, q0, q12 - vbit.8 q15, q1, q12 - vbit.8 q4, q2, q12 - ldr r8, [r0], #4 - vshl.u8 q0, q14, #2 - vshl.u8 q1, q15, #2 - vshl.u8 q2, q4, #2 - ldr r12, [r0], #4 - vshr.u8 q14, q14, #2 - vshr.u8 q15, q15, #2 - vshr.u8 q4, q4, #2 - rbit lr, lr - vbit.8 q0, q14, q5 - rbit r4, r4 - vbit.8 q1, q15, q5 - rbit r8, r8 - vbit.8 q2, q4, q5 - rbit r12, r12 - vshl.u8 q14, q0, #4 - rev lr, lr - vshl.u8 q15, q1, #4 - rev r4, r4 - vshl.u8 q4, q2, #4 - rev r8, r8 - vsri.u8 q14, q0, #4 - rev r12, r12 - vsri.u8 q15, q1, #4 - vsri.u8 q4, q2, #4 - vmov d10, lr, r4 - vmov d11, r8, r12 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d11, d16 - vmull.p64 q7, d10, d17 - vmull.p64 q0, d10, d16 - vmull.p64 q1, d11, d17 - veor.8 q2, q2, q7 - # X += C * H^2 - vmull.p64 q3, d8, d18 - vmull.p64 q7, d9, d19 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d9, d18 - vmull.p64 q7, d8, d19 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # X += C * H^3 - vmull.p64 q3, d30, d20 - vmull.p64 q7, d31, d21 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d31, d20 - vmull.p64 q7, d30, d21 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # X += C * H^4 - vmull.p64 q3, d24, d22 - vmull.p64 q7, d25, d23 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d25, d22 - vmull.p64 q7, d24, d23 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH - sub r10, r10, #4 - cmp r10, #4 - bge L_aes_gcm_decrypt_arm32_crypto_in_start_4 - cmp r10, #1 - blt L_aes_gcm_decrypt_arm32_crypto_in_done - beq L_aes_gcm_decrypt_arm32_crypto_in_start_1 -L_aes_gcm_decrypt_arm32_crypto_in_start_2: - vld1.32 {q14-q15}, [r0]! - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshl.u8 q1, q15, #1 - vshr.u8 q14, q14, #1 - vshr.u8 q15, q15, #1 - vbif.8 q14, q0, q12 - vbif.8 q15, q1, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshl.u8 q1, q15, #2 - vshr.u8 q14, q14, #2 - vshr.u8 q15, q15, #2 - vbit.8 q0, q14, q12 - vbit.8 q1, q15, q12 - vshl.u8 q14, q0, #4 - vshl.u8 q15, q1, #4 - vsri.u8 q14, q0, #4 - vsri.u8 q15, q1, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d31, d16 - vmull.p64 q7, d30, d17 - vmull.p64 q0, d30, d16 - vmull.p64 q1, d31, d17 - veor.8 q2, q2, q7 - # X += C * H^2 - vmull.p64 q3, d24, d18 - vmull.p64 q7, d25, d19 - veor.8 q0, q0, q3 - veor.8 q1, q1, q7 - vmull.p64 q3, d25, d18 - vmull.p64 q7, d24, d19 - veor.8 q2, q2, q3 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH - sub r10, r10, #2 - cmp r10, #0 - beq L_aes_gcm_decrypt_arm32_crypto_in_done -L_aes_gcm_decrypt_arm32_crypto_in_start_1: - vld1.32 {q14}, [r0]! - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q7, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH -L_aes_gcm_decrypt_arm32_crypto_in_done: - ands r11, r2, #15 - beq L_aes_gcm_decrypt_arm32_crypto_in_partial_done - veor.8 q0, q0, q0 - mov r12, r11 - vst1.32 {q0}, [r9] - cmp r12, #4 - blt L_aes_gcm_decrypt_arm32_crypto_in_start_sw -L_aes_gcm_decrypt_arm32_crypto_in_start_dw: - ldr r8, [r0], #4 - sub r12, r12, #4 - str r8, [r9], #4 - cmp r12, #4 - bge L_aes_gcm_decrypt_arm32_crypto_in_start_dw -L_aes_gcm_decrypt_arm32_crypto_in_start_sw: - cmp r12, #2 - blt L_aes_gcm_decrypt_arm32_crypto_in_start_byte - ldrh r8, [r0], #2 - sub r12, r12, #2 - strh r8, [r9], #2 -L_aes_gcm_decrypt_arm32_crypto_in_start_byte: - cmp r12, #1 - blt L_aes_gcm_decrypt_arm32_crypto_in_end_bytes - ldrb r8, [r0], #1 - subs r12, r12, #1 - strb r8, [r9], #1 - bne L_aes_gcm_decrypt_arm32_crypto_in_start_byte -L_aes_gcm_decrypt_arm32_crypto_in_end_bytes: - sub r9, r9, r11 - vld1.32 {q14}, [r9] - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q7, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q7, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q7 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - # Done GHASH -L_aes_gcm_decrypt_arm32_crypto_in_partial_done: - sub r0, r0, r2 -L_aes_gcm_decrypt_arm32_crypto_done_gcm: - # nonceSz - ldr r12, [sp, #100] - # Load Nonce - cmp r12, #12 - bne L_aes_gcm_decrypt_arm32_crypto_ghash_nonce - ldr r5, [r3] - ldr r8, [r3, #4] - ldr r12, [r3, #8] - vmov.i32 q6, #0x1000000 - vmov.32 s24, r5 - vmov.32 s25, r8 - vmov.32 s26, r12 - mov r5, #1 - b L_aes_gcm_decrypt_arm32_crypto_done_nonce -L_aes_gcm_decrypt_arm32_crypto_ghash_nonce: - lsr r10, r12, #4 - cmp r10, #0 - beq L_aes_gcm_decrypt_arm32_crypto_nonce_done -L_aes_gcm_decrypt_arm32_crypto_nonce_start_1: - vld1.32 {q14}, [r3]! - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q6, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q6, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q6 - # Reduce - vmull.p64 q6, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d13 - veor.8 d1, d1, d12 - vmull.p64 q6, d2, d26 - veor.8 q6, q6, q0 - # Done GHASH - subs r10, r10, #1 - bne L_aes_gcm_decrypt_arm32_crypto_nonce_start_1 -L_aes_gcm_decrypt_arm32_crypto_nonce_done: - ands r11, r12, #15 - beq L_aes_gcm_decrypt_arm32_crypto_nonce_partial_done - veor.8 q0, q0, q0 - mov r12, r11 - vst1.32 {q0}, [r9] - cmp r12, #4 - blt L_aes_gcm_decrypt_arm32_crypto_nonce_start_sw -L_aes_gcm_decrypt_arm32_crypto_nonce_start_dw: - ldr r8, [r3], #4 - sub r12, r12, #4 - str r8, [r9], #4 - cmp r12, #4 - bge L_aes_gcm_decrypt_arm32_crypto_nonce_start_dw -L_aes_gcm_decrypt_arm32_crypto_nonce_start_sw: - cmp r12, #2 - blt L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte - ldrh r8, [r3], #2 - sub r12, r12, #2 - strh r8, [r9], #2 -L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte: - cmp r12, #1 - blt L_aes_gcm_decrypt_arm32_crypto_nonce_end_bytes - ldrb r8, [r3], #1 - subs r12, r12, #1 - strb r8, [r9], #1 - bne L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte -L_aes_gcm_decrypt_arm32_crypto_nonce_end_bytes: - sub r9, r9, r11 - vld1.32 {q14}, [r9] - vmov.i8 q12, #0x55 - vshl.u8 q0, q14, #1 - vshr.u8 q14, q14, #1 - vbif.8 q14, q0, q12 - vmov.i8 q12, #51 - vshl.u8 q0, q14, #2 - vshr.u8 q14, q14, #2 - vbit.8 q0, q14, q12 - vshl.u8 q14, q0, #4 - vsri.u8 q14, q0, #4 - veor.8 q12, q6, q14 - # X = C * H^1 - vmull.p64 q2, d25, d16 - vmull.p64 q6, d24, d17 - vmull.p64 q0, d24, d16 - vmull.p64 q1, d25, d17 - veor.8 q2, q2, q6 - # Reduce - vmull.p64 q6, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d13 - veor.8 d1, d1, d12 - vmull.p64 q6, d2, d26 - veor.8 q6, q6, q0 - # Done GHASH -L_aes_gcm_decrypt_arm32_crypto_nonce_partial_done: - veor.8 q0, q0, q0 - # nonceSz - ldr r12, [sp, #100] - lsr r10, r12, #29 - lsl r11, r12, #3 - rbit r10, r10 - rbit r11, r11 - vmov.32 s2, r10 - vmov.32 s3, r11 - veor.8 q6, q6, q0 - vmull.p64 q2, d13, d16 - vmull.p64 q12, d12, d17 - vmull.p64 q0, d12, d16 - vmull.p64 q1, d13, d17 - veor.8 q2, q2, q12 - vmull.p64 q6, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d13 - veor.8 d1, d1, d12 - vmull.p64 q6, d2, d26 - veor.8 q6, q6, q0 - vmov.32 r5, s24 - vmov.32 r8, s25 - vmov.32 r12, s26 - rbit r5, r5 - rbit r8, r8 - rbit r12, r12 - rev r5, r5 - rev r8, r8 - rev r12, r12 - vmov.32 s24, r5 - vmov.32 s25, r8 - vmov.32 s26, r12 - vmov.32 r5, s27 - rbit r5, r5 - rev r5, r5 - vmov.32 s27, r5 - rev r5, r5 -L_aes_gcm_decrypt_arm32_crypto_done_nonce: - # reg - ldr r9, [sp, #132] - vst1.32 {q7}, [r9] - # tmp - ldr r9, [sp, #128] - vldm.32 r7!, {q0-q3} - vldm.32 r7!, {q7-q13} - # nr - ldr r12, [sp, #136] - lsr r10, r2, #4 - and r11, r2, #15 - cmp r12, #12 - blt L_aes_gcm_decrypt_arm32_crypto_start_128 - bgt L_aes_gcm_decrypt_arm32_crypto_start_256 - # AES_GCM_192 -#ifndef NO_AES_192 - cmp r10, #1 - blt L_aes_gcm_decrypt_arm32_crypto_192_done - beq L_aes_gcm_decrypt_arm32_crypto_192_start_1 -L_aes_gcm_decrypt_arm32_crypto_192_start_2: - add r8, r5, #1 - vmov.8 q4, q6 - add r5, r5, #2 - vmov.8 q5, q6 - rev r8, r8 - rev r12, r5 - vmov s19, r8 - vmov s23, r12 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q5, q0 - aesmc.8 q5, q5 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q5, q1 - aesmc.8 q5, q5 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q5, q2 - aesmc.8 q5, q5 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q5, q3 - aesmc.8 q5, q5 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q5, q7 - aesmc.8 q5, q5 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q5, q8 - aesmc.8 q5, q5 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q5, q9 - aesmc.8 q5, q5 - subs r10, r10, #2 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q5, q10 - aesmc.8 q5, q5 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q5, q11 - aesmc.8 q5, q5 - aese.8 q4, q12 - aesmc.8 q4, q4 - aese.8 q5, q12 - aesmc.8 q5, q5 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - aese.8 q5, q13 - aesmc.8 q5, q5 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - aese.8 q5, q14 - veor.8 q5, q5, q15 - vld1.8 {q14-q15}, [r0]! - sub r7, r7, #16 - veor.8 q14, q14, q4 - veor.8 q15, q15, q5 - vst1.8 {q14-q15}, [r1]! - cmp r10, #1 - bgt L_aes_gcm_decrypt_arm32_crypto_192_start_2 - blt L_aes_gcm_decrypt_arm32_crypto_192_done -L_aes_gcm_decrypt_arm32_crypto_192_start_1: - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q4, q12 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - vld1.8 {q14}, [r0]! - sub r7, r7, #16 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r1]! -L_aes_gcm_decrypt_arm32_crypto_192_done: - ands r11, r2, #15 - beq L_aes_gcm_decrypt_arm32_crypto_192_partial_done - veor.8 q14, q14, q14 - mov r4, r11 - vst1.32 {q14}, [r9] - cmp r4, #4 - blt L_aes_gcm_decrypt_arm32_crypto_192_start_sw -L_aes_gcm_decrypt_arm32_crypto_192_start_dw: - ldr lr, [r0], #4 - sub r4, r4, #4 - str lr, [r9], #4 - cmp r4, #4 - bge L_aes_gcm_decrypt_arm32_crypto_192_start_dw -L_aes_gcm_decrypt_arm32_crypto_192_start_sw: - cmp r4, #2 - blt L_aes_gcm_decrypt_arm32_crypto_192_start_byte - ldrh lr, [r0], #2 - sub r4, r4, #2 - strh lr, [r9], #2 -L_aes_gcm_decrypt_arm32_crypto_192_start_byte: - cmp r4, #1 - blt L_aes_gcm_decrypt_arm32_crypto_192_end_bytes - ldrb lr, [r0], #1 - subs r4, r4, #1 - strb lr, [r9], #1 - bne L_aes_gcm_decrypt_arm32_crypto_192_start_byte -L_aes_gcm_decrypt_arm32_crypto_192_end_bytes: - sub r9, r9, r11 - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q4, q12 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - vld1.8 {q14}, [r9] - sub r7, r7, #16 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r9] - mov r4, r11 - cmp r4, #4 - blt L_aes_gcm_decrypt_arm32_crypto_192_out_start_sw -L_aes_gcm_decrypt_arm32_crypto_192_out_start_dw: - ldr lr, [r9], #4 - sub r4, r4, #4 - str lr, [r1], #4 - cmp r4, #4 - bge L_aes_gcm_decrypt_arm32_crypto_192_out_start_dw -L_aes_gcm_decrypt_arm32_crypto_192_out_start_sw: - cmp r4, #2 - blt L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte - ldrh lr, [r9], #2 - sub r4, r4, #2 - strh lr, [r1], #2 -L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte: - cmp r4, #1 - blt L_aes_gcm_decrypt_arm32_crypto_192_out_end_bytes - ldrb lr, [r9], #1 - subs r4, r4, #1 - strb lr, [r1], #1 - bne L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte -L_aes_gcm_decrypt_arm32_crypto_192_out_end_bytes: -L_aes_gcm_decrypt_arm32_crypto_192_partial_done: - # Finish - add r8, r2, #15 - sub r8, r5, r8, lsr #4 - rev r8, r8 - vmov.32 s27, r8 - aese.8 q6, q0 - aesmc.8 q6, q6 - aese.8 q6, q1 - aesmc.8 q6, q6 - aese.8 q6, q2 - aesmc.8 q6, q6 - aese.8 q6, q3 - aesmc.8 q6, q6 - aese.8 q6, q7 - aesmc.8 q6, q6 - aese.8 q6, q8 - aesmc.8 q6, q6 - aese.8 q6, q9 - aesmc.8 q6, q6 - aese.8 q6, q10 - aesmc.8 q6, q6 - aese.8 q6, q11 - aesmc.8 q6, q6 - aese.8 q6, q12 - aesmc.8 q6, q6 - vld1.32 {q14}, [r7]! - aese.8 q6, q13 - aesmc.8 q6, q6 - vld1.32 {q15}, [r7] - aese.8 q6, q14 - veor.8 q6, q6, q15 - sub r7, r7, #16 -#endif /* !NO_AES_192 */ - b L_aes_gcm_decrypt_arm32_crypto_done_enc - # AES_GCM_256 -L_aes_gcm_decrypt_arm32_crypto_start_256: -#ifndef NO_AES_256 - cmp r10, #1 - blt L_aes_gcm_decrypt_arm32_crypto_256_done - beq L_aes_gcm_decrypt_arm32_crypto_256_start_1 -L_aes_gcm_decrypt_arm32_crypto_256_start_2: - add r8, r5, #1 - vmov.8 q4, q6 - add r5, r5, #2 - vmov.8 q5, q6 - rev r8, r8 - rev r12, r5 - vmov s19, r8 - vmov s23, r12 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q5, q0 - aesmc.8 q5, q5 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q5, q1 - aesmc.8 q5, q5 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q5, q2 - aesmc.8 q5, q5 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q5, q3 - aesmc.8 q5, q5 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q5, q7 - aesmc.8 q5, q5 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q5, q8 - aesmc.8 q5, q5 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q5, q9 - aesmc.8 q5, q5 - subs r10, r10, #2 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q5, q10 - aesmc.8 q5, q5 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q5, q11 - aesmc.8 q5, q5 - aese.8 q4, q12 - aesmc.8 q4, q4 - aese.8 q5, q12 - aesmc.8 q5, q5 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - aese.8 q5, q13 - aesmc.8 q5, q5 - vld1.32 {q15}, [r7]! - aese.8 q4, q14 - aesmc.8 q4, q4 - aese.8 q5, q14 - aesmc.8 q5, q5 - vld1.32 {q14}, [r7]! - aese.8 q4, q15 - aesmc.8 q4, q4 - aese.8 q5, q15 - aesmc.8 q5, q5 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - aese.8 q5, q14 - veor.8 q5, q5, q15 - vld1.8 {q14-q15}, [r0]! - sub r7, r7, #48 - veor.8 q14, q14, q4 - veor.8 q15, q15, q5 - vst1.8 {q14-q15}, [r1]! - cmp r10, #1 - bgt L_aes_gcm_decrypt_arm32_crypto_256_start_2 - blt L_aes_gcm_decrypt_arm32_crypto_256_done -L_aes_gcm_decrypt_arm32_crypto_256_start_1: - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q4, q12 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7]! - aese.8 q4, q14 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q15 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - vld1.8 {q14}, [r0]! - sub r7, r7, #48 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r1]! -L_aes_gcm_decrypt_arm32_crypto_256_done: - ands r11, r2, #15 - beq L_aes_gcm_decrypt_arm32_crypto_256_partial_done - veor.8 q14, q14, q14 - mov r4, r11 - vst1.32 {q14}, [r9] - cmp r4, #4 - blt L_aes_gcm_decrypt_arm32_crypto_256_start_sw -L_aes_gcm_decrypt_arm32_crypto_256_start_dw: - ldr lr, [r0], #4 - sub r4, r4, #4 - str lr, [r9], #4 - cmp r4, #4 - bge L_aes_gcm_decrypt_arm32_crypto_256_start_dw -L_aes_gcm_decrypt_arm32_crypto_256_start_sw: - cmp r4, #2 - blt L_aes_gcm_decrypt_arm32_crypto_256_start_byte - ldrh lr, [r0], #2 - sub r4, r4, #2 - strh lr, [r9], #2 -L_aes_gcm_decrypt_arm32_crypto_256_start_byte: - cmp r4, #1 - blt L_aes_gcm_decrypt_arm32_crypto_256_end_bytes - ldrb lr, [r0], #1 - subs r4, r4, #1 - strb lr, [r9], #1 - bne L_aes_gcm_decrypt_arm32_crypto_256_start_byte -L_aes_gcm_decrypt_arm32_crypto_256_end_bytes: - sub r9, r9, r11 - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q4, q12 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q13 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7]! - aese.8 q4, q14 - aesmc.8 q4, q4 - vld1.32 {q14}, [r7]! - aese.8 q4, q15 - aesmc.8 q4, q4 - vld1.32 {q15}, [r7] - aese.8 q4, q14 - veor.8 q4, q4, q15 - vld1.8 {q14}, [r9] - sub r7, r7, #48 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r9] - mov r4, r11 - cmp r4, #4 - blt L_aes_gcm_decrypt_arm32_crypto_256_out_start_sw -L_aes_gcm_decrypt_arm32_crypto_256_out_start_dw: - ldr lr, [r9], #4 - sub r4, r4, #4 - str lr, [r1], #4 - cmp r4, #4 - bge L_aes_gcm_decrypt_arm32_crypto_256_out_start_dw -L_aes_gcm_decrypt_arm32_crypto_256_out_start_sw: - cmp r4, #2 - blt L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte - ldrh lr, [r9], #2 - sub r4, r4, #2 - strh lr, [r1], #2 -L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte: - cmp r4, #1 - blt L_aes_gcm_decrypt_arm32_crypto_256_out_end_bytes - ldrb lr, [r9], #1 - subs r4, r4, #1 - strb lr, [r1], #1 - bne L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte -L_aes_gcm_decrypt_arm32_crypto_256_out_end_bytes: -L_aes_gcm_decrypt_arm32_crypto_256_partial_done: - # Finish - add r8, r2, #15 - sub r8, r5, r8, lsr #4 - rev r8, r8 - vmov.32 s27, r8 - aese.8 q6, q0 - aesmc.8 q6, q6 - aese.8 q6, q1 - aesmc.8 q6, q6 - aese.8 q6, q2 - aesmc.8 q6, q6 - aese.8 q6, q3 - aesmc.8 q6, q6 - aese.8 q6, q7 - aesmc.8 q6, q6 - aese.8 q6, q8 - aesmc.8 q6, q6 - aese.8 q6, q9 - aesmc.8 q6, q6 - aese.8 q6, q10 - aesmc.8 q6, q6 - aese.8 q6, q11 - aesmc.8 q6, q6 - aese.8 q6, q12 - aesmc.8 q6, q6 - vld1.32 {q14}, [r7]! - aese.8 q6, q13 - aesmc.8 q6, q6 - vld1.32 {q15}, [r7]! - aese.8 q6, q14 - aesmc.8 q6, q6 - vld1.32 {q14}, [r7]! - aese.8 q6, q15 - aesmc.8 q6, q6 - vld1.32 {q15}, [r7] - aese.8 q6, q14 - veor.8 q6, q6, q15 - sub r7, r7, #48 -#endif /* !NO_AES_256 */ - b L_aes_gcm_decrypt_arm32_crypto_done_enc - # AES_GCM_128 -L_aes_gcm_decrypt_arm32_crypto_start_128: -#ifndef NO_AES_128 - cmp r10, #1 - blt L_aes_gcm_decrypt_arm32_crypto_128_done - beq L_aes_gcm_decrypt_arm32_crypto_128_start_1 -L_aes_gcm_decrypt_arm32_crypto_128_start_2: - add r8, r5, #1 - vmov.8 q4, q6 - add r5, r5, #2 - vmov.8 q5, q6 - rev r8, r8 - rev r12, r5 - vmov s19, r8 - vmov s23, r12 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q5, q0 - aesmc.8 q5, q5 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q5, q1 - aesmc.8 q5, q5 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q5, q2 - aesmc.8 q5, q5 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q5, q3 - aesmc.8 q5, q5 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q5, q7 - aesmc.8 q5, q5 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q5, q8 - aesmc.8 q5, q5 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q5, q9 - aesmc.8 q5, q5 - subs r10, r10, #2 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q5, q10 - aesmc.8 q5, q5 - aese.8 q4, q11 - aesmc.8 q4, q4 - aese.8 q5, q11 - aesmc.8 q5, q5 - vld1.8 {q14-q15}, [r0]! - aese.8 q4, q12 - veor.8 q4, q4, q13 - aese.8 q5, q12 - veor.8 q5, q5, q13 - veor.8 q14, q14, q4 - veor.8 q15, q15, q5 - vst1.8 {q14-q15}, [r1]! - cmp r10, #1 - bgt L_aes_gcm_decrypt_arm32_crypto_128_start_2 - blt L_aes_gcm_decrypt_arm32_crypto_128_done -L_aes_gcm_decrypt_arm32_crypto_128_start_1: - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - vld1.8 {q14}, [r0]! - aese.8 q4, q12 - veor.8 q4, q4, q13 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r1]! -L_aes_gcm_decrypt_arm32_crypto_128_done: - ands r11, r2, #15 - beq L_aes_gcm_decrypt_arm32_crypto_128_partial_done - veor.8 q14, q14, q14 - mov r4, r11 - vst1.32 {q14}, [r9] - cmp r4, #4 - blt L_aes_gcm_decrypt_arm32_crypto_128_start_sw -L_aes_gcm_decrypt_arm32_crypto_128_start_dw: - ldr lr, [r0], #4 - sub r4, r4, #4 - str lr, [r9], #4 - cmp r4, #4 - bge L_aes_gcm_decrypt_arm32_crypto_128_start_dw -L_aes_gcm_decrypt_arm32_crypto_128_start_sw: - cmp r4, #2 - blt L_aes_gcm_decrypt_arm32_crypto_128_start_byte - ldrh lr, [r0], #2 - sub r4, r4, #2 - strh lr, [r9], #2 -L_aes_gcm_decrypt_arm32_crypto_128_start_byte: - cmp r4, #1 - blt L_aes_gcm_decrypt_arm32_crypto_128_end_bytes - ldrb lr, [r0], #1 - subs r4, r4, #1 - strb lr, [r9], #1 - bne L_aes_gcm_decrypt_arm32_crypto_128_start_byte -L_aes_gcm_decrypt_arm32_crypto_128_end_bytes: - sub r9, r9, r11 - add r5, r5, #1 - vmov.8 q4, q6 - rev r8, r5 - vmov s19, r8 - aese.8 q4, q0 - aesmc.8 q4, q4 - aese.8 q4, q1 - aesmc.8 q4, q4 - aese.8 q4, q2 - aesmc.8 q4, q4 - aese.8 q4, q3 - aesmc.8 q4, q4 - aese.8 q4, q7 - aesmc.8 q4, q4 - aese.8 q4, q8 - aesmc.8 q4, q4 - aese.8 q4, q9 - aesmc.8 q4, q4 - aese.8 q4, q10 - aesmc.8 q4, q4 - aese.8 q4, q11 - aesmc.8 q4, q4 - vld1.8 {q14}, [r9] - aese.8 q4, q12 - veor.8 q4, q4, q13 - veor.8 q14, q14, q4 - vst1.32 {q14}, [r9] - mov r4, r11 - cmp r4, #4 - blt L_aes_gcm_decrypt_arm32_crypto_128_out_start_sw -L_aes_gcm_decrypt_arm32_crypto_128_out_start_dw: - ldr lr, [r9], #4 - sub r4, r4, #4 - str lr, [r1], #4 - cmp r4, #4 - bge L_aes_gcm_decrypt_arm32_crypto_128_out_start_dw -L_aes_gcm_decrypt_arm32_crypto_128_out_start_sw: - cmp r4, #2 - blt L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte - ldrh lr, [r9], #2 - sub r4, r4, #2 - strh lr, [r1], #2 -L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte: - cmp r4, #1 - blt L_aes_gcm_decrypt_arm32_crypto_128_out_end_bytes - ldrb lr, [r9], #1 - subs r4, r4, #1 - strb lr, [r1], #1 - bne L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte -L_aes_gcm_decrypt_arm32_crypto_128_out_end_bytes: -L_aes_gcm_decrypt_arm32_crypto_128_partial_done: - # Finish - add r8, r2, #15 - sub r8, r5, r8, lsr #4 - rev r8, r8 - vmov.32 s27, r8 - aese.8 q6, q0 - aesmc.8 q6, q6 - aese.8 q6, q1 - aesmc.8 q6, q6 - aese.8 q6, q2 - aesmc.8 q6, q6 - aese.8 q6, q3 - aesmc.8 q6, q6 - aese.8 q6, q7 - aesmc.8 q6, q6 - aese.8 q6, q8 - aesmc.8 q6, q6 - aese.8 q6, q9 - aesmc.8 q6, q6 - aese.8 q6, q10 - aesmc.8 q6, q6 - aese.8 q6, q11 - aesmc.8 q6, q6 - aese.8 q6, q12 - veor.8 q6, q6, q13 -#endif /* !NO_AES_128 */ -L_aes_gcm_decrypt_arm32_crypto_done_enc: - vmov.i8 q13, #0x87 - vshr.u64 q13, q13, #56 - # gcm_h - ldr r8, [sp, #124] - vld1.32 {q8}, [r8] - # reg - ldr r9, [sp, #132] - vld1.32 {q7}, [r9] - # tmp - ldr r9, [sp, #128] - lsr lr, r6, #29 - lsl r6, r6, #3 - rbit lr, lr - rbit r6, r6 - vmov s0, lr - vmov s1, r6 - lsr lr, r2, #29 - lsl r2, r2, #3 - rbit lr, lr - rbit r2, r2 - vmov s2, lr - vmov s3, r2 - veor.8 q7, q7, q0 - vmull.p64 q2, d15, d16 - vmull.p64 q3, d14, d17 - vmull.p64 q0, d14, d16 - vmull.p64 q1, d15, d17 - veor.8 q2, q2, q3 - # Reduce - vmull.p64 q7, d3, d27 - veor.8 d2, d2, d5 - veor.8 d1, d1, d4 - veor.8 d2, d2, d15 - veor.8 d1, d1, d14 - vmull.p64 q7, d2, d26 - veor.8 q7, q7, q0 - vmov.i8 q0, #0x55 - vshl.u8 q1, q7, #1 - vshr.u8 q7, q7, #1 - vbif.8 q7, q1, q0 - vmov.i8 q0, #51 - vshl.u8 q1, q7, #2 - vshr.u8 q7, q7, #2 - vbit.8 q1, q7, q0 - vshl.u8 q7, q1, #4 - vsri.u8 q7, q1, #4 - # DONE - veor.8 q7, q7, q6 - # tag - ldr lr, [sp, #104] - # tagSz - ldr r4, [sp, #108] - cmp r4, #16 - blt L_aes_gcm_decrypt_arm32_crypto_tag_part_tag - vld1.8 {q0}, [lr] - b L_aes_gcm_decrypt_arm32_crypto_tag_tag_loaded -L_aes_gcm_decrypt_arm32_crypto_tag_part_tag: - veor.8 q0, q0, q0 - mov r12, r4 - vst1.32 {q0}, [r9] - cmp r12, #4 - blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_sw -L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_dw: - ldr r8, [lr], #4 - sub r12, r12, #4 - str r8, [r9], #4 - cmp r12, #4 - bge L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_dw -L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_sw: - cmp r12, #2 - blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte - ldrh r8, [lr], #2 - sub r12, r12, #2 - strh r8, [r9], #2 -L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte: - cmp r12, #1 - blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_end_bytes - ldrb r8, [lr], #1 - subs r12, r12, #1 - strb r8, [r9], #1 - bne L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte -L_aes_gcm_decrypt_arm32_crypto_tag_tag_end_bytes: - sub r9, r9, r4 - vld1.32 {q0}, [r9] - mov r12, #16 - vst1.32 {q7}, [r9] - sub r12, r12, r4 - eor r8, r8, r8 - add r9, r9, r4 -L_aes_gcm_decrypt_arm32_crypto_tag_calc_tag_byte: - strb r8, [r9], #1 - subs r12, r12, #1 - bne L_aes_gcm_decrypt_arm32_crypto_tag_calc_tag_byte - subs r9, r9, #16 - vld1.32 {q7}, [r9] -L_aes_gcm_decrypt_arm32_crypto_tag_tag_loaded: - vceq.i32 q0, q0, q7 - vmov r5, s0 - vmov r8, s1 - and r12, r5, r8 - vmov r5, s2 - vmov r8, s3 - and r12, r12, r5 - and r12, r12, r8 - mov r5, #-180 - mvn r12, r12 - and r0, r5, r12 - vpop {d8-d15} - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size AES_GCM_decrypt_AARCH32,.-AES_GCM_decrypt_AARCH32 -#endif /* HAVE_AES_DECRYPT */ -#endif /* HAVE_AESGCM */ -#ifdef WOLFSSL_AES_XTS - .text - .align 4 - .globl AES_XTS_encrypt_AARCH32 - .type AES_XTS_encrypt_AARCH32, %function -AES_XTS_encrypt_AARCH32: - push {r4, r5, r6, r7, r8, r9, lr} - vpush {d8-d15} - ldr r12, [sp, #92] - ldr lr, [sp, #96] - ldr r5, [sp, #104] - vldm.32 lr!, {q2-q9} - lsr r4, r2, #4 - and r2, r2, #15 - vld1.8 {q0}, [r3] - cmp r5, #12 - blt L_aes_xts_encrypt_arm32_crypto_start_128 - bgt L_aes_xts_encrypt_arm32_crypto_start_256 - # AES_XTS_192 -#ifndef NO_AES_192 - vldm.32 lr!, {q10-q14} - aese.8 q0, q2 - aesmc.8 q0, q0 - aese.8 q0, q3 - aesmc.8 q0, q0 - aese.8 q0, q4 - aesmc.8 q0, q0 - aese.8 q0, q5 - aesmc.8 q0, q0 - aese.8 q0, q6 - aesmc.8 q0, q0 - aese.8 q0, q7 - aesmc.8 q0, q0 - aese.8 q0, q8 - aesmc.8 q0, q0 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q0, q11 - aesmc.8 q0, q0 - aese.8 q0, q12 - aesmc.8 q0, q0 - aese.8 q0, q13 - veor.32 q0, q0, q14 - vmov r6, r7, d0 - vmov r8, r9, d1 - vldm.32 r12!, {q2-q9} - vldm.32 r12!, {q10-q14} - mov lr, #0x87 - cmp r4, #1 - blt L_aes_xts_encrypt_arm32_crypto_192_done -L_aes_xts_encrypt_arm32_crypto_192_start_1: - vld1.8 {q1}, [r0]! - veor.32 q1, q1, q0 - aese.8 q1, q2 - aesmc.8 q1, q1 - and r5, lr, r9, asr #31 - aese.8 q1, q3 - aesmc.8 q1, q1 - lsl r9, r9, #1 - aese.8 q1, q4 - aesmc.8 q1, q1 - orr r9, r9, r8, lsr #31 - aese.8 q1, q5 - aesmc.8 q1, q1 - lsl r8, r8, #1 - aese.8 q1, q6 - aesmc.8 q1, q1 - orr r8, r8, r7, lsr #31 - aese.8 q1, q7 - aesmc.8 q1, q1 - lsl r7, r7, #1 - aese.8 q1, q8 - aesmc.8 q1, q1 - orr r7, r7, r6, lsr #31 - aese.8 q1, q9 - aesmc.8 q1, q1 - eor r6, r5, r6, lsl #1 - aese.8 q1, q10 - aesmc.8 q1, q1 - aese.8 q1, q11 - aesmc.8 q1, q1 - aese.8 q1, q12 - aesmc.8 q1, q1 - aese.8 q1, q13 - veor.32 q1, q1, q14 - veor.32 q1, q1, q0 - vmov d0, r6, r7 - vmov d1, r8, r9 - subs r4, r4, #1 - vst1.8 {q1}, [r1]! - bne L_aes_xts_encrypt_arm32_crypto_192_start_1 -L_aes_xts_encrypt_arm32_crypto_192_done: - cmp r2, #0 - beq L_aes_xts_encrypt_arm32_crypto_192_partial_done - sub r1, r1, #16 - ldr r4, [sp, #100] - vld1.8 {q1}, [r1]! - vst1.32 {q1}, [r4] - mov r5, r2 -L_aes_xts_encrypt_arm32_crypto_192_start_byte: - ldrb r8, [r4] - ldrb r9, [r0], #1 - strb r8, [r1], #1 - strb r9, [r4], #1 - subs r5, r5, #1 - bgt L_aes_xts_encrypt_arm32_crypto_192_start_byte - sub r1, r1, r2 - sub r4, r4, r2 - sub r1, r1, #16 - vld1.32 {q1}, [r4] - veor.32 q1, q1, q0 - aese.8 q1, q2 - aesmc.8 q1, q1 - aese.8 q1, q3 - aesmc.8 q1, q1 - aese.8 q1, q4 - aesmc.8 q1, q1 - aese.8 q1, q5 - aesmc.8 q1, q1 - aese.8 q1, q6 - aesmc.8 q1, q1 - aese.8 q1, q7 - aesmc.8 q1, q1 - aese.8 q1, q8 - aesmc.8 q1, q1 - aese.8 q1, q9 - aesmc.8 q1, q1 - aese.8 q1, q10 - aesmc.8 q1, q1 - aese.8 q1, q11 - aesmc.8 q1, q1 - aese.8 q1, q12 - aesmc.8 q1, q1 - aese.8 q1, q13 - veor.32 q1, q1, q14 - veor.32 q1, q1, q0 - vst1.8 {q1}, [r1] -L_aes_xts_encrypt_arm32_crypto_192_partial_done: -#endif /* !NO_AES_192 */ - b L_aes_xts_encrypt_arm32_crypto_done - # AES_XTS_256 -L_aes_xts_encrypt_arm32_crypto_start_256: -#ifndef NO_AES_256 - vldm.32 lr!, {q10-q13} - aese.8 q0, q2 - aesmc.8 q0, q0 - aese.8 q0, q3 - aesmc.8 q0, q0 - aese.8 q0, q4 - aesmc.8 q0, q0 - aese.8 q0, q5 - aesmc.8 q0, q0 - aese.8 q0, q6 - aesmc.8 q0, q0 - aese.8 q0, q7 - aesmc.8 q0, q0 - aese.8 q0, q8 - aesmc.8 q0, q0 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q0, q11 - aesmc.8 q0, q0 - aese.8 q0, q12 - aesmc.8 q0, q0 - vld1.32 {q14}, [lr]! - aese.8 q0, q13 - aesmc.8 q0, q0 - vld1.32 {q15}, [lr]! - aese.8 q0, q14 - aesmc.8 q0, q0 - vld1.32 {q14}, [lr]! - aese.8 q0, q15 - veor.32 q0, q0, q14 - vmov r6, r7, d0 - vmov r8, r9, d1 - vldm.32 r12!, {q2-q9} - vldm.32 r12!, {q10-q13} - mov lr, #0x87 - cmp r4, #1 - blt L_aes_xts_encrypt_arm32_crypto_256_done -L_aes_xts_encrypt_arm32_crypto_256_start_1: - vld1.8 {q1}, [r0]! - veor.32 q1, q1, q0 - aese.8 q1, q2 - aesmc.8 q1, q1 - and r5, lr, r9, asr #31 - aese.8 q1, q3 - aesmc.8 q1, q1 - lsl r9, r9, #1 - aese.8 q1, q4 - aesmc.8 q1, q1 - orr r9, r9, r8, lsr #31 - aese.8 q1, q5 - aesmc.8 q1, q1 - lsl r8, r8, #1 - aese.8 q1, q6 - aesmc.8 q1, q1 - orr r8, r8, r7, lsr #31 - aese.8 q1, q7 - aesmc.8 q1, q1 - lsl r7, r7, #1 - aese.8 q1, q8 - aesmc.8 q1, q1 - orr r7, r7, r6, lsr #31 - aese.8 q1, q9 - aesmc.8 q1, q1 - eor r6, r5, r6, lsl #1 - aese.8 q1, q10 - aesmc.8 q1, q1 - aese.8 q1, q11 - aesmc.8 q1, q1 - aese.8 q1, q12 - aesmc.8 q1, q1 - vld1.32 {q14}, [r12]! - aese.8 q1, q13 - aesmc.8 q1, q1 - vld1.32 {q15}, [r12]! - aese.8 q1, q14 - aesmc.8 q1, q1 - vld1.32 {q14}, [r12]! - aese.8 q1, q15 - veor.32 q1, q1, q14 - sub r12, r12, #48 - veor.32 q1, q1, q0 - vmov d0, r6, r7 - vmov d1, r8, r9 - subs r4, r4, #1 - vst1.8 {q1}, [r1]! - bne L_aes_xts_encrypt_arm32_crypto_256_start_1 -L_aes_xts_encrypt_arm32_crypto_256_done: - cmp r2, #0 - beq L_aes_xts_encrypt_arm32_crypto_256_partial_done - sub r1, r1, #16 - ldr r4, [sp, #100] - vld1.8 {q1}, [r1]! - vst1.32 {q1}, [r4] - mov r5, r2 -L_aes_xts_encrypt_arm32_crypto_256_start_byte: - ldrb r8, [r4] - ldrb r9, [r0], #1 - strb r8, [r1], #1 - strb r9, [r4], #1 - subs r5, r5, #1 - bgt L_aes_xts_encrypt_arm32_crypto_256_start_byte - sub r1, r1, r2 - sub r4, r4, r2 - sub r1, r1, #16 - vld1.32 {q1}, [r4] - veor.32 q1, q1, q0 - aese.8 q1, q2 - aesmc.8 q1, q1 - aese.8 q1, q3 - aesmc.8 q1, q1 - aese.8 q1, q4 - aesmc.8 q1, q1 - aese.8 q1, q5 - aesmc.8 q1, q1 - aese.8 q1, q6 - aesmc.8 q1, q1 - aese.8 q1, q7 - aesmc.8 q1, q1 - aese.8 q1, q8 - aesmc.8 q1, q1 - aese.8 q1, q9 - aesmc.8 q1, q1 - aese.8 q1, q10 - aesmc.8 q1, q1 - aese.8 q1, q11 - aesmc.8 q1, q1 - aese.8 q1, q12 - aesmc.8 q1, q1 - vld1.32 {q14}, [r12]! - aese.8 q1, q13 - aesmc.8 q1, q1 - vld1.32 {q15}, [r12]! - aese.8 q1, q14 - aesmc.8 q1, q1 - vld1.32 {q14}, [r12]! - aese.8 q1, q15 - veor.32 q1, q1, q14 - veor.32 q1, q1, q0 - vst1.8 {q1}, [r1] -L_aes_xts_encrypt_arm32_crypto_256_partial_done: -#endif /* !NO_AES_256 */ - b L_aes_xts_encrypt_arm32_crypto_done - # AES_XTS_128 -L_aes_xts_encrypt_arm32_crypto_start_128: -#ifndef NO_AES_128 - vldm.32 lr!, {q10-q12} - aese.8 q0, q2 - aesmc.8 q0, q0 - aese.8 q0, q3 - aesmc.8 q0, q0 - aese.8 q0, q4 - aesmc.8 q0, q0 - aese.8 q0, q5 - aesmc.8 q0, q0 - aese.8 q0, q6 - aesmc.8 q0, q0 - aese.8 q0, q7 - aesmc.8 q0, q0 - aese.8 q0, q8 - aesmc.8 q0, q0 - aese.8 q0, q9 - aesmc.8 q0, q0 - aese.8 q0, q10 - aesmc.8 q0, q0 - aese.8 q0, q11 - veor.32 q0, q0, q12 - vmov r6, r7, d0 - vmov r8, r9, d1 - vldm.32 r12!, {q2-q9} - vldm.32 r12!, {q10-q12} - mov lr, #0x87 - cmp r4, #1 - blt L_aes_xts_encrypt_arm32_crypto_128_done -L_aes_xts_encrypt_arm32_crypto_128_start_1: - vld1.8 {q1}, [r0]! - veor.32 q1, q1, q0 - aese.8 q1, q2 - aesmc.8 q1, q1 - and r5, lr, r9, asr #31 - aese.8 q1, q3 - aesmc.8 q1, q1 - lsl r9, r9, #1 - aese.8 q1, q4 - aesmc.8 q1, q1 - orr r9, r9, r8, lsr #31 - aese.8 q1, q5 - aesmc.8 q1, q1 - lsl r8, r8, #1 - aese.8 q1, q6 - aesmc.8 q1, q1 - orr r8, r8, r7, lsr #31 - aese.8 q1, q7 - aesmc.8 q1, q1 - lsl r7, r7, #1 - aese.8 q1, q8 - aesmc.8 q1, q1 - orr r7, r7, r6, lsr #31 - aese.8 q1, q9 - aesmc.8 q1, q1 - eor r6, r5, r6, lsl #1 - aese.8 q1, q10 - aesmc.8 q1, q1 - aese.8 q1, q11 - veor.32 q1, q1, q12 - veor.32 q1, q1, q0 - vmov d0, r6, r7 - vmov d1, r8, r9 - subs r4, r4, #1 - vst1.8 {q1}, [r1]! - bne L_aes_xts_encrypt_arm32_crypto_128_start_1 -L_aes_xts_encrypt_arm32_crypto_128_done: - cmp r2, #0 - beq L_aes_xts_encrypt_arm32_crypto_128_partial_done - sub r1, r1, #16 - ldr r4, [sp, #100] - vld1.8 {q1}, [r1]! - vst1.32 {q1}, [r4] - mov r5, r2 -L_aes_xts_encrypt_arm32_crypto_128_start_byte: - ldrb r8, [r4] - ldrb r9, [r0], #1 - strb r8, [r1], #1 - strb r9, [r4], #1 - subs r5, r5, #1 - bgt L_aes_xts_encrypt_arm32_crypto_128_start_byte - sub r1, r1, r2 - sub r4, r4, r2 - sub r1, r1, #16 - vld1.32 {q1}, [r4] - veor.32 q1, q1, q0 - aese.8 q1, q2 - aesmc.8 q1, q1 - aese.8 q1, q3 - aesmc.8 q1, q1 - aese.8 q1, q4 - aesmc.8 q1, q1 - aese.8 q1, q5 - aesmc.8 q1, q1 - aese.8 q1, q6 - aesmc.8 q1, q1 - aese.8 q1, q7 - aesmc.8 q1, q1 - aese.8 q1, q8 - aesmc.8 q1, q1 - aese.8 q1, q9 - aesmc.8 q1, q1 - aese.8 q1, q10 - aesmc.8 q1, q1 - aese.8 q1, q11 - veor.32 q1, q1, q12 - veor.32 q1, q1, q0 - vst1.8 {q1}, [r1] -L_aes_xts_encrypt_arm32_crypto_128_partial_done: -#endif /* !NO_AES_128 */ -L_aes_xts_encrypt_arm32_crypto_done: - vpop {d8-d15} - pop {r4, r5, r6, r7, r8, r9, pc} - .size AES_XTS_encrypt_AARCH32,.-AES_XTS_encrypt_AARCH32 -#ifdef HAVE_AES_DECRYPT - .text - .align 4 - .globl AES_XTS_decrypt_AARCH32 - .type AES_XTS_decrypt_AARCH32, %function -AES_XTS_decrypt_AARCH32: - push {r4, r5, r6, r7, r8, r9, lr} - vpush {d8-d15} - ldr r12, [sp, #92] - ldr lr, [sp, #96] - vldm.32 lr!, {q2-q9} - eor r5, r5, r5 - lsr r4, r2, #4 - ands r2, r2, #15 - sub r5, r5, r2 - sub r4, r4, r5, lsr #31 - vld1.8 {q1}, [r3] - ldr r5, [sp, #104] - cmp r5, #12 - blt L_aes_xts_decrypt_arm32_crypto_start_128 - bgt L_aes_xts_decrypt_arm32_crypto_start_256 - # AES_XTS_192 -#ifndef NO_AES_192 - vldm.32 lr!, {q10-q14} - aese.8 q1, q2 - aesmc.8 q1, q1 - aese.8 q1, q3 - aesmc.8 q1, q1 - aese.8 q1, q4 - aesmc.8 q1, q1 - aese.8 q1, q5 - aesmc.8 q1, q1 - aese.8 q1, q6 - aesmc.8 q1, q1 - aese.8 q1, q7 - aesmc.8 q1, q1 - aese.8 q1, q8 - aesmc.8 q1, q1 - aese.8 q1, q9 - aesmc.8 q1, q1 - aese.8 q1, q10 - aesmc.8 q1, q1 - aese.8 q1, q11 - aesmc.8 q1, q1 - aese.8 q1, q12 - aesmc.8 q1, q1 - aese.8 q1, q13 - veor.32 q1, q1, q14 - vmov r6, r7, d2 - vmov r8, r9, d3 - vldm.32 r12!, {q2-q9} - vldm.32 r12!, {q10-q14} - mov lr, #0x87 - cmp r4, #1 - blt L_aes_xts_decrypt_arm32_crypto_192_done -L_aes_xts_decrypt_arm32_crypto_192_start_1: - vld1.8 {q0}, [r0]! - veor.32 q0, q0, q1 - aesd.8 q0, q2 - aesimc.8 q0, q0 - and r5, lr, r9, asr #31 - aesd.8 q0, q3 - aesimc.8 q0, q0 - lsl r9, r9, #1 - aesd.8 q0, q4 - aesimc.8 q0, q0 - orr r9, r9, r8, lsr #31 - aesd.8 q0, q5 - aesimc.8 q0, q0 - lsl r8, r8, #1 - aesd.8 q0, q6 - aesimc.8 q0, q0 - orr r8, r8, r7, lsr #31 - aesd.8 q0, q7 - aesimc.8 q0, q0 - lsl r7, r7, #1 - aesd.8 q0, q8 - aesimc.8 q0, q0 - orr r7, r7, r6, lsr #31 - aesd.8 q0, q9 - aesimc.8 q0, q0 - eor r6, r5, r6, lsl #1 - aesd.8 q0, q10 - aesimc.8 q0, q0 - aesd.8 q0, q11 - aesimc.8 q0, q0 - aesd.8 q0, q12 - aesimc.8 q0, q0 - aesd.8 q0, q13 - veor.32 q0, q0, q14 - veor.32 q0, q0, q1 - vmov d2, r6, r7 - vmov d3, r8, r9 - subs r4, r4, #1 - vst1.8 {q0}, [r1]! - bne L_aes_xts_decrypt_arm32_crypto_192_start_1 -L_aes_xts_decrypt_arm32_crypto_192_done: - cmp r2, #0 - beq L_aes_xts_decrypt_arm32_crypto_192_partial_done - and r5, lr, r9, asr #31 - lsl r9, r9, #1 - orr r9, r9, r8, lsr #31 - lsl r8, r8, #1 - orr r8, r8, r7, lsr #31 - lsl r7, r7, #1 - orr r7, r7, r6, lsr #31 - eor r6, r5, r6, lsl #1 - vmov d30, r6, r7 - vmov d31, r8, r9 - vld1.8 {q0}, [r0]! - ldr r4, [sp, #100] - veor.32 q0, q0, q15 - aesd.8 q0, q2 - aesimc.8 q0, q0 - aesd.8 q0, q3 - aesimc.8 q0, q0 - aesd.8 q0, q4 - aesimc.8 q0, q0 - aesd.8 q0, q5 - aesimc.8 q0, q0 - aesd.8 q0, q6 - aesimc.8 q0, q0 - aesd.8 q0, q7 - aesimc.8 q0, q0 - aesd.8 q0, q8 - aesimc.8 q0, q0 - aesd.8 q0, q9 - aesimc.8 q0, q0 - aesd.8 q0, q10 - aesimc.8 q0, q0 - aesd.8 q0, q11 - aesimc.8 q0, q0 - aesd.8 q0, q12 - aesimc.8 q0, q0 - aesd.8 q0, q13 - veor.32 q0, q0, q14 - veor.32 q0, q0, q15 - vst1.32 {q0}, [r4] - add r1, r1, #16 - mov r5, r2 -L_aes_xts_decrypt_arm32_crypto_192_start_byte: - ldrb r8, [r4] - ldrb r9, [r0], #1 - strb r8, [r1], #1 - strb r9, [r4], #1 - subs r5, r5, #1 - bgt L_aes_xts_decrypt_arm32_crypto_192_start_byte - sub r1, r1, r2 - sub r4, r4, r2 - sub r1, r1, #16 - vld1.32 {q0}, [r4] - veor.32 q0, q0, q1 - aesd.8 q0, q2 - aesimc.8 q0, q0 - aesd.8 q0, q3 - aesimc.8 q0, q0 - aesd.8 q0, q4 - aesimc.8 q0, q0 - aesd.8 q0, q5 - aesimc.8 q0, q0 - aesd.8 q0, q6 - aesimc.8 q0, q0 - aesd.8 q0, q7 - aesimc.8 q0, q0 - aesd.8 q0, q8 - aesimc.8 q0, q0 - aesd.8 q0, q9 - aesimc.8 q0, q0 - aesd.8 q0, q10 - aesimc.8 q0, q0 - aesd.8 q0, q11 - aesimc.8 q0, q0 - aesd.8 q0, q12 - aesimc.8 q0, q0 - aesd.8 q0, q13 - veor.32 q0, q0, q14 - veor.32 q0, q0, q1 - vst1.8 {q0}, [r1] -L_aes_xts_decrypt_arm32_crypto_192_partial_done: -#endif /* !NO_AES_192 */ - b L_aes_xts_decrypt_arm32_crypto_done - # AES_XTS_256 -L_aes_xts_decrypt_arm32_crypto_start_256: -#ifndef NO_AES_256 - vldm.32 lr!, {q10-q13} - aese.8 q1, q2 - aesmc.8 q1, q1 - aese.8 q1, q3 - aesmc.8 q1, q1 - aese.8 q1, q4 - aesmc.8 q1, q1 - aese.8 q1, q5 - aesmc.8 q1, q1 - aese.8 q1, q6 - aesmc.8 q1, q1 - aese.8 q1, q7 - aesmc.8 q1, q1 - aese.8 q1, q8 - aesmc.8 q1, q1 - aese.8 q1, q9 - aesmc.8 q1, q1 - aese.8 q1, q10 - aesmc.8 q1, q1 - aese.8 q1, q11 - aesmc.8 q1, q1 - aese.8 q1, q12 - aesmc.8 q1, q1 - vld1.32 {q14}, [lr]! - aese.8 q1, q13 - aesmc.8 q1, q1 - vld1.32 {q15}, [lr]! - aese.8 q1, q14 - aesmc.8 q1, q1 - vld1.32 {q14}, [lr]! - aese.8 q1, q15 - veor.32 q1, q1, q14 - vmov r6, r7, d2 - vmov r8, r9, d3 - vldm.32 r12!, {q2-q9} - vldm.32 r12!, {q10-q12} - mov lr, #0x87 - cmp r4, #1 - blt L_aes_xts_decrypt_arm32_crypto_256_done -L_aes_xts_decrypt_arm32_crypto_256_start_1: - vld1.8 {q0}, [r0]! - veor.32 q0, q0, q1 - aesd.8 q0, q2 - aesimc.8 q0, q0 - and r5, lr, r9, asr #31 - aesd.8 q0, q3 - aesimc.8 q0, q0 - lsl r9, r9, #1 - aesd.8 q0, q4 - aesimc.8 q0, q0 - orr r9, r9, r8, lsr #31 - aesd.8 q0, q5 - aesimc.8 q0, q0 - lsl r8, r8, #1 - aesd.8 q0, q6 - aesimc.8 q0, q0 - orr r8, r8, r7, lsr #31 - aesd.8 q0, q7 - aesimc.8 q0, q0 - lsl r7, r7, #1 - aesd.8 q0, q8 - aesimc.8 q0, q0 - orr r7, r7, r6, lsr #31 - aesd.8 q0, q9 - aesimc.8 q0, q0 - eor r6, r5, r6, lsl #1 - aesd.8 q0, q10 - aesimc.8 q0, q0 - aesd.8 q0, q11 - aesimc.8 q0, q0 - vld1.32 {q13}, [r12]! - aesd.8 q0, q12 - aesimc.8 q0, q0 - vld1.32 {q14}, [r12]! - aesd.8 q0, q13 - aesimc.8 q0, q0 - vld1.32 {q13}, [r12]! - aesd.8 q0, q14 - aesimc.8 q0, q0 - vld1.32 {q14}, [r12] - aesd.8 q0, q13 - veor.32 q0, q0, q14 - sub r12, r12, #48 - veor.32 q0, q0, q1 - vmov d2, r6, r7 - vmov d3, r8, r9 - subs r4, r4, #1 - vst1.8 {q0}, [r1]! - bne L_aes_xts_decrypt_arm32_crypto_256_start_1 -L_aes_xts_decrypt_arm32_crypto_256_done: - cmp r2, #0 - beq L_aes_xts_decrypt_arm32_crypto_256_partial_done - and r5, lr, r9, asr #31 - lsl r9, r9, #1 - orr r9, r9, r8, lsr #31 - lsl r8, r8, #1 - orr r8, r8, r7, lsr #31 - lsl r7, r7, #1 - orr r7, r7, r6, lsr #31 - eor r6, r5, r6, lsl #1 - vmov d30, r6, r7 - vmov d31, r8, r9 - vld1.8 {q0}, [r0]! - ldr r4, [sp, #100] - veor.32 q0, q0, q15 - aesd.8 q0, q2 - aesimc.8 q0, q0 - aesd.8 q0, q3 - aesimc.8 q0, q0 - aesd.8 q0, q4 - aesimc.8 q0, q0 - aesd.8 q0, q5 - aesimc.8 q0, q0 - aesd.8 q0, q6 - aesimc.8 q0, q0 - aesd.8 q0, q7 - aesimc.8 q0, q0 - aesd.8 q0, q8 - aesimc.8 q0, q0 - aesd.8 q0, q9 - aesimc.8 q0, q0 - aesd.8 q0, q10 - aesimc.8 q0, q0 - aesd.8 q0, q11 - aesimc.8 q0, q0 - vld1.32 {q13}, [r12]! - aesd.8 q0, q12 - aesimc.8 q0, q0 - vld1.32 {q14}, [r12]! - aesd.8 q0, q13 - aesimc.8 q0, q0 - vld1.32 {q13}, [r12]! - aesd.8 q0, q14 - aesimc.8 q0, q0 - vld1.32 {q14}, [r12] - aesd.8 q0, q13 - veor.32 q0, q0, q14 - sub r12, r12, #48 - veor.32 q0, q0, q15 - vst1.32 {q0}, [r4] - add r1, r1, #16 - mov r5, r2 -L_aes_xts_decrypt_arm32_crypto_256_start_byte: - ldrb r8, [r4] - ldrb r9, [r0], #1 - strb r8, [r1], #1 - strb r9, [r4], #1 - subs r5, r5, #1 - bgt L_aes_xts_decrypt_arm32_crypto_256_start_byte - sub r1, r1, r2 - sub r4, r4, r2 - sub r1, r1, #16 - vld1.32 {q0}, [r4] - veor.32 q0, q0, q1 - aesd.8 q0, q2 - aesimc.8 q0, q0 - aesd.8 q0, q3 - aesimc.8 q0, q0 - aesd.8 q0, q4 - aesimc.8 q0, q0 - aesd.8 q0, q5 - aesimc.8 q0, q0 - aesd.8 q0, q6 - aesimc.8 q0, q0 - aesd.8 q0, q7 - aesimc.8 q0, q0 - aesd.8 q0, q8 - aesimc.8 q0, q0 - aesd.8 q0, q9 - aesimc.8 q0, q0 - aesd.8 q0, q10 - aesimc.8 q0, q0 - aesd.8 q0, q11 - aesimc.8 q0, q0 - vld1.32 {q13}, [r12]! - aesd.8 q0, q12 - aesimc.8 q0, q0 - vld1.32 {q14}, [r12]! - aesd.8 q0, q13 - aesimc.8 q0, q0 - vld1.32 {q13}, [r12]! - aesd.8 q0, q14 - aesimc.8 q0, q0 - vld1.32 {q14}, [r12] - aesd.8 q0, q13 - veor.32 q0, q0, q14 - veor.32 q0, q0, q1 - vst1.8 {q0}, [r1] -L_aes_xts_decrypt_arm32_crypto_256_partial_done: -#endif /* !NO_AES_256 */ - b L_aes_xts_decrypt_arm32_crypto_done - # AES_XTS_128 -L_aes_xts_decrypt_arm32_crypto_start_128: -#ifndef NO_AES_128 - vldm.32 lr!, {q10-q12} - aese.8 q1, q2 - aesmc.8 q1, q1 - aese.8 q1, q3 - aesmc.8 q1, q1 - aese.8 q1, q4 - aesmc.8 q1, q1 - aese.8 q1, q5 - aesmc.8 q1, q1 - aese.8 q1, q6 - aesmc.8 q1, q1 - aese.8 q1, q7 - aesmc.8 q1, q1 - aese.8 q1, q8 - aesmc.8 q1, q1 - aese.8 q1, q9 - aesmc.8 q1, q1 - aese.8 q1, q10 - aesmc.8 q1, q1 - aese.8 q1, q11 - veor.32 q1, q1, q12 - vmov r6, r7, d2 - vmov r8, r9, d3 - vldm.32 r12!, {q2-q9} - vldm.32 r12!, {q10-q12} - mov lr, #0x87 - cmp r4, #1 - blt L_aes_xts_decrypt_arm32_crypto_128_done -L_aes_xts_decrypt_arm32_crypto_128_start_1: - vld1.8 {q0}, [r0]! - veor.32 q0, q0, q1 - aesd.8 q0, q2 - aesimc.8 q0, q0 - and r5, lr, r9, asr #31 - aesd.8 q0, q3 - aesimc.8 q0, q0 - lsl r9, r9, #1 - aesd.8 q0, q4 - aesimc.8 q0, q0 - orr r9, r9, r8, lsr #31 - aesd.8 q0, q5 - aesimc.8 q0, q0 - lsl r8, r8, #1 - aesd.8 q0, q6 - aesimc.8 q0, q0 - orr r8, r8, r7, lsr #31 - aesd.8 q0, q7 - aesimc.8 q0, q0 - lsl r7, r7, #1 - aesd.8 q0, q8 - aesimc.8 q0, q0 - orr r7, r7, r6, lsr #31 - aesd.8 q0, q9 - aesimc.8 q0, q0 - eor r6, r5, r6, lsl #1 - aesd.8 q0, q10 - aesimc.8 q0, q0 - aesd.8 q0, q11 - veor.32 q0, q0, q12 - veor.32 q0, q0, q1 - vmov d2, r6, r7 - vmov d3, r8, r9 - subs r4, r4, #1 - vst1.8 {q0}, [r1]! - bne L_aes_xts_decrypt_arm32_crypto_128_start_1 -L_aes_xts_decrypt_arm32_crypto_128_done: - cmp r2, #0 - beq L_aes_xts_decrypt_arm32_crypto_128_partial_done - and r5, lr, r9, asr #31 - lsl r9, r9, #1 - orr r9, r9, r8, lsr #31 - lsl r8, r8, #1 - orr r8, r8, r7, lsr #31 - lsl r7, r7, #1 - orr r7, r7, r6, lsr #31 - eor r6, r5, r6, lsl #1 - vmov d30, r6, r7 - vmov d31, r8, r9 - vld1.8 {q0}, [r0]! - ldr r4, [sp, #100] - veor.32 q0, q0, q15 - aesd.8 q0, q2 - aesimc.8 q0, q0 - aesd.8 q0, q3 - aesimc.8 q0, q0 - aesd.8 q0, q4 - aesimc.8 q0, q0 - aesd.8 q0, q5 - aesimc.8 q0, q0 - aesd.8 q0, q6 - aesimc.8 q0, q0 - aesd.8 q0, q7 - aesimc.8 q0, q0 - aesd.8 q0, q8 - aesimc.8 q0, q0 - aesd.8 q0, q9 - aesimc.8 q0, q0 - aesd.8 q0, q10 - aesimc.8 q0, q0 - aesd.8 q0, q11 - veor.32 q0, q0, q12 - veor.32 q0, q0, q15 - vst1.32 {q0}, [r4] - add r1, r1, #16 - mov r5, r2 -L_aes_xts_decrypt_arm32_crypto_128_start_byte: - ldrb r8, [r4] - ldrb r9, [r0], #1 - strb r8, [r1], #1 - strb r9, [r4], #1 - subs r5, r5, #1 - bgt L_aes_xts_decrypt_arm32_crypto_128_start_byte - sub r1, r1, r2 - sub r4, r4, r2 - sub r1, r1, #16 - vld1.32 {q0}, [r4] - veor.32 q0, q0, q1 - aesd.8 q0, q2 - aesimc.8 q0, q0 - aesd.8 q0, q3 - aesimc.8 q0, q0 - aesd.8 q0, q4 - aesimc.8 q0, q0 - aesd.8 q0, q5 - aesimc.8 q0, q0 - aesd.8 q0, q6 - aesimc.8 q0, q0 - aesd.8 q0, q7 - aesimc.8 q0, q0 - aesd.8 q0, q8 - aesimc.8 q0, q0 - aesd.8 q0, q9 - aesimc.8 q0, q0 - aesd.8 q0, q10 - aesimc.8 q0, q0 - aesd.8 q0, q11 - veor.32 q0, q0, q12 - veor.32 q0, q0, q1 - vst1.8 {q0}, [r1] -L_aes_xts_decrypt_arm32_crypto_128_partial_done: -#endif /* !NO_AES_128 */ -L_aes_xts_decrypt_arm32_crypto_done: - vpop {d8-d15} - pop {r4, r5, r6, r7, r8, r9, pc} - .size AES_XTS_decrypt_AARCH32,.-AES_XTS_decrypt_AARCH32 -#endif /* HAVE_AES_DECRYPT */ -#endif /* WOLFSSL_AES_XTS */ -#ifdef WOLFSSL_AESGCM_SIV - .text - .align 4 - .globl AES_GCMSIV_polyval_crypto - .type AES_GCMSIV_polyval_crypto, %function -AES_GCMSIV_polyval_crypto: - vpush {d8-d15} - veor.8 q2, q2, q2 - vld1.8 {q1}, [r1] - vld1.8 {q0}, [r0] - vrev64.8 q0, q0 - vext.8 q0, q0, q0, #8 - vmull.p64 q7, d2, d2 - vmull.p64 q11, d2, d3 - vmull.p64 q12, d3, d2 - vmull.p64 q8, d3, d3 - veor.8 q11, q11, q12 - vext.8 q12, q2, q11, #8 - vext.8 q11, q11, q2, #8 - veor.8 q7, q7, q12 - veor.8 q8, q8, q11 - vshr.u32 q6, q7, #31 - vshr.u32 q9, q8, #31 - vshl.i32 q7, q7, #1 - vshl.i32 q8, q8, #1 - vext.8 q10, q6, q2, #12 - vext.8 q9, q2, q9, #12 - vext.8 q6, q2, q6, #12 - veor.8 q7, q7, q6 - veor.8 q8, q8, q9 - veor.8 q8, q8, q10 - vshl.i32 q6, q7, #31 - vshl.i32 q9, q7, #30 - vshl.i32 q10, q7, #25 - veor.8 q6, q6, q9 - veor.8 q6, q6, q10 - vext.8 q9, q6, q2, #4 - vext.8 q6, q2, q6, #4 - veor.8 q7, q7, q6 - vshr.u32 q11, q7, #1 - vshr.u32 q12, q7, #2 - vshr.u32 q13, q7, #7 - veor.8 q11, q11, q12 - veor.8 q11, q11, q13 - veor.8 q11, q11, q9 - veor.8 q7, q7, q11 - veor.8 q3, q8, q7 - vmull.p64 q7, d6, d2 - vmull.p64 q11, d6, d3 - vmull.p64 q12, d7, d2 - vmull.p64 q8, d7, d3 - veor.8 q11, q11, q12 - vext.8 q12, q2, q11, #8 - vext.8 q11, q11, q2, #8 - veor.8 q7, q7, q12 - veor.8 q8, q8, q11 - vshr.u32 q6, q7, #31 - vshr.u32 q9, q8, #31 - vshl.i32 q7, q7, #1 - vshl.i32 q8, q8, #1 - vext.8 q10, q6, q2, #12 - vext.8 q9, q2, q9, #12 - vext.8 q6, q2, q6, #12 - veor.8 q7, q7, q6 - veor.8 q8, q8, q9 - veor.8 q8, q8, q10 - vshl.i32 q6, q7, #31 - vshl.i32 q9, q7, #30 - vshl.i32 q10, q7, #25 - veor.8 q6, q6, q9 - veor.8 q6, q6, q10 - vext.8 q9, q6, q2, #4 - vext.8 q6, q2, q6, #4 - veor.8 q7, q7, q6 - vshr.u32 q11, q7, #1 - vshr.u32 q12, q7, #2 - vshr.u32 q13, q7, #7 - veor.8 q11, q11, q12 - veor.8 q11, q11, q13 - veor.8 q11, q11, q9 - veor.8 q7, q7, q11 - veor.8 q4, q8, q7 - vmull.p64 q7, d6, d6 - vmull.p64 q11, d6, d7 - vmull.p64 q12, d7, d6 - vmull.p64 q8, d7, d7 - veor.8 q11, q11, q12 - vext.8 q12, q2, q11, #8 - vext.8 q11, q11, q2, #8 - veor.8 q7, q7, q12 - veor.8 q8, q8, q11 - vshr.u32 q6, q7, #31 - vshr.u32 q9, q8, #31 - vshl.i32 q7, q7, #1 - vshl.i32 q8, q8, #1 - vext.8 q10, q6, q2, #12 - vext.8 q9, q2, q9, #12 - vext.8 q6, q2, q6, #12 - veor.8 q7, q7, q6 - veor.8 q8, q8, q9 - veor.8 q8, q8, q10 - vshl.i32 q6, q7, #31 - vshl.i32 q9, q7, #30 - vshl.i32 q10, q7, #25 - veor.8 q6, q6, q9 - veor.8 q6, q6, q10 - vext.8 q9, q6, q2, #4 - vext.8 q6, q2, q6, #4 - veor.8 q7, q7, q6 - vshr.u32 q11, q7, #1 - vshr.u32 q12, q7, #2 - vshr.u32 q13, q7, #7 - veor.8 q11, q11, q12 - veor.8 q11, q11, q13 - veor.8 q11, q11, q9 - veor.8 q7, q7, q11 - veor.8 q5, q8, q7 - lsr r12, r3, #2 - cmp r12, #0 - beq L_AES_GCMSIV_polyval_crypto_rem_start -L_AES_GCMSIV_polyval_crypto_group: - vld1.8 {q6}, [r2]! - veor.8 q6, q6, q0 - vmull.p64 q9, d12, d10 - vmull.p64 q11, d12, d11 - vmull.p64 q12, d13, d10 - vmull.p64 q10, d13, d11 - veor.8 q11, q11, q12 - vext.8 q12, q2, q11, #8 - vext.8 q11, q11, q2, #8 - veor.8 q9, q9, q12 - veor.8 q10, q10, q11 - vld1.8 {q6}, [r2]! - vmull.p64 q7, d12, d8 - vmull.p64 q11, d12, d9 - vmull.p64 q12, d13, d8 - vmull.p64 q8, d13, d9 - veor.8 q11, q11, q12 - vext.8 q12, q2, q11, #8 - vext.8 q11, q11, q2, #8 - veor.8 q7, q7, q12 - veor.8 q8, q8, q11 - veor.8 q9, q9, q7 - veor.8 q10, q10, q8 - vld1.8 {q6}, [r2]! - vmull.p64 q7, d12, d6 - vmull.p64 q11, d12, d7 - vmull.p64 q12, d13, d6 - vmull.p64 q8, d13, d7 - veor.8 q11, q11, q12 - vext.8 q12, q2, q11, #8 - vext.8 q11, q11, q2, #8 - veor.8 q7, q7, q12 - veor.8 q8, q8, q11 - veor.8 q9, q9, q7 - veor.8 q10, q10, q8 - vld1.8 {q6}, [r2]! - vmull.p64 q7, d12, d2 - vmull.p64 q11, d12, d3 - vmull.p64 q12, d13, d2 - vmull.p64 q8, d13, d3 - veor.8 q11, q11, q12 - vext.8 q12, q2, q11, #8 - vext.8 q11, q11, q2, #8 - veor.8 q7, q7, q12 - veor.8 q8, q8, q11 - veor.8 q9, q9, q7 - veor.8 q10, q10, q8 - vshr.u32 q6, q9, #31 - vshr.u32 q7, q10, #31 - vshl.i32 q9, q9, #1 - vshl.i32 q10, q10, #1 - vext.8 q8, q6, q2, #12 - vext.8 q7, q2, q7, #12 - vext.8 q6, q2, q6, #12 - veor.8 q9, q9, q6 - veor.8 q10, q10, q7 - veor.8 q10, q10, q8 - vshl.i32 q6, q9, #31 - vshl.i32 q7, q9, #30 - vshl.i32 q8, q9, #25 - veor.8 q6, q6, q7 - veor.8 q6, q6, q8 - vext.8 q7, q6, q2, #4 - vext.8 q6, q2, q6, #4 - veor.8 q9, q9, q6 - vshr.u32 q11, q9, #1 - vshr.u32 q12, q9, #2 - vshr.u32 q13, q9, #7 - veor.8 q11, q11, q12 - veor.8 q11, q11, q13 - veor.8 q11, q11, q7 - veor.8 q9, q9, q11 - veor.8 q0, q10, q9 - subs r12, r12, #1 - bne L_AES_GCMSIV_polyval_crypto_group -L_AES_GCMSIV_polyval_crypto_rem_start: - and r3, r3, #3 - cmp r3, #0 - beq L_AES_GCMSIV_polyval_crypto_done -L_AES_GCMSIV_polyval_crypto_rem: - vld1.8 {q6}, [r2]! - veor.8 q0, q0, q6 - vmull.p64 q9, d0, d2 - vmull.p64 q11, d0, d3 - vmull.p64 q12, d1, d2 - vmull.p64 q10, d1, d3 - veor.8 q11, q11, q12 - vext.8 q12, q2, q11, #8 - vext.8 q11, q11, q2, #8 - veor.8 q9, q9, q12 - veor.8 q10, q10, q11 - vshr.u32 q6, q9, #31 - vshr.u32 q7, q10, #31 - vshl.i32 q9, q9, #1 - vshl.i32 q10, q10, #1 - vext.8 q8, q6, q2, #12 - vext.8 q7, q2, q7, #12 - vext.8 q6, q2, q6, #12 - veor.8 q9, q9, q6 - veor.8 q10, q10, q7 - veor.8 q10, q10, q8 - vshl.i32 q6, q9, #31 - vshl.i32 q7, q9, #30 - vshl.i32 q8, q9, #25 - veor.8 q6, q6, q7 - veor.8 q6, q6, q8 - vext.8 q7, q6, q2, #4 - vext.8 q6, q2, q6, #4 - veor.8 q9, q9, q6 - vshr.u32 q11, q9, #1 - vshr.u32 q12, q9, #2 - vshr.u32 q13, q9, #7 - veor.8 q11, q11, q12 - veor.8 q11, q11, q13 - veor.8 q11, q11, q7 - veor.8 q9, q9, q11 - veor.8 q0, q10, q9 - subs r3, r3, #1 - bne L_AES_GCMSIV_polyval_crypto_rem -L_AES_GCMSIV_polyval_crypto_done: - vrev64.8 q0, q0 - vext.8 q0, q0, q0, #8 - vst1.8 {q0}, [r0] - vpop {d8-d15} - bx lr - .size AES_GCMSIV_polyval_crypto,.-AES_GCMSIV_polyval_crypto - .text - .align 4 - .globl AES_GCMSIV_ctr_crypto - .type AES_GCMSIV_ctr_crypto, %function -AES_GCMSIV_ctr_crypto: - push {r4, r5, r6, r7, r8, r9, r10, lr} - vpush {d8-d11} - ldr r12, [sp, #64] - ldr lr, [sp, #68] - vld1.8 {q0}, [lr] - lsr r5, r2, #4 - vmov r7, r8, d0 - vmov r9, r10, d1 -L_AES_GCMSIV_ctr_crypto_loop2: - cmp r5, #2 - blt L_AES_GCMSIV_ctr_crypto_tail - vmov d2, r7, r8 - vmov d3, r9, r10 - add r2, r7, #1 - vmov d4, r2, r8 - vmov d5, r9, r10 - add r7, r7, #2 - mov r4, r3 - sub r6, r12, #1 -L_AES_GCMSIV_ctr_crypto_rounds2: - vld1.32 {q5}, [r4]! - aese.8 q1, q5 - aesmc.8 q1, q1 - aese.8 q2, q5 - aesmc.8 q2, q2 - subs r6, r6, #1 - bne L_AES_GCMSIV_ctr_crypto_rounds2 - vld1.32 {q5}, [r4]! - aese.8 q1, q5 - aese.8 q2, q5 - vld1.32 {q5}, [r4] - veor.8 q1, q1, q5 - veor.8 q2, q2, q5 - vld1.8 {q3-q4}, [r0]! - veor.8 q3, q3, q1 - veor.8 q4, q4, q2 - vst1.8 {q3-q4}, [r1]! - sub r5, r5, #2 - b L_AES_GCMSIV_ctr_crypto_loop2 -L_AES_GCMSIV_ctr_crypto_tail: - cmp r5, #0 - beq L_AES_GCMSIV_ctr_crypto_done - vmov d2, r7, r8 - vmov d3, r9, r10 - add r7, r7, #1 - mov r4, r3 - sub r6, r12, #1 -L_AES_GCMSIV_ctr_crypto_rounds1: - vld1.32 {q5}, [r4]! - aese.8 q1, q5 - aesmc.8 q1, q1 - subs r6, r6, #1 - bne L_AES_GCMSIV_ctr_crypto_rounds1 - vld1.32 {q5}, [r4]! - aese.8 q1, q5 - vld1.32 {q5}, [r4] - veor.8 q1, q1, q5 - vld1.8 {q3}, [r0]! - veor.8 q3, q3, q1 - vst1.8 {q3}, [r1]! -L_AES_GCMSIV_ctr_crypto_done: - vmov d0, r7, r8 - vst1.8 {q0}, [lr] - vpop {d8-d11} - pop {r4, r5, r6, r7, r8, r9, r10, pc} - .size AES_GCMSIV_ctr_crypto,.-AES_GCMSIV_ctr_crypto -#endif /* WOLFSSL_AESGCM_SIV */ -#else -#ifdef HAVE_AES_DECRYPT -#ifndef __APPLE__ - .text - .type L_AES_ARM32_td_data, %object - .size L_AES_ARM32_td_data, 1024 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 -#else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_ARM32_td_data: - .long 0x5051f4a7,0x537e4165,0xc31a17a4,0x963a275e - .long 0xcb3bab6b,0xf11f9d45,0xabacfa58,0x934be303 - .long 0x552030fa,0xf6ad766d,0x9188cc76,0x25f5024c - .long 0xfc4fe5d7,0xd7c52acb,0x80263544,0x8fb562a3 - .long 0x49deb15a,0x6725ba1b,0x9845ea0e,0xe15dfec0 - .long 0x02c32f75,0x12814cf0,0xa38d4697,0xc66bd3f9 - .long 0xe7038f5f,0x9515929c,0xebbf6d7a,0xda955259 - .long 0x2dd4be83,0xd3587421,0x2949e069,0x448ec9c8 - .long 0x6a75c289,0x78f48e79,0x6b99583e,0xdd27b971 - .long 0xb6bee14f,0x17f088ad,0x66c920ac,0xb47dce3a - .long 0x1863df4a,0x82e51a31,0x60975133,0x4562537f - .long 0xe0b16477,0x84bb6bae,0x1cfe81a0,0x94f9082b - .long 0x58704868,0x198f45fd,0x8794de6c,0xb7527bf8 - .long 0x23ab73d3,0xe2724b02,0x57e31f8f,0x2a6655ab - .long 0x07b2eb28,0x032fb5c2,0x9a86c57b,0xa5d33708 - .long 0xf2302887,0xb223bfa5,0xba02036a,0x5ced1682 - .long 0x2b8acf1c,0x92a779b4,0xf0f307f2,0xa14e69e2 - .long 0xcd65daf4,0xd50605be,0x1fd13462,0x8ac4a6fe - .long 0x9d342e53,0xa0a2f355,0x32058ae1,0x75a4f6eb - .long 0x390b83ec,0xaa4060ef,0x065e719f,0x51bd6e10 - .long 0xf93e218a,0x3d96dd06,0xaedd3e05,0x464de6bd - .long 0xb591548d,0x0571c45d,0x6f0406d4,0xff605015 - .long 0x241998fb,0x97d6bde9,0xcc894043,0x7767d99e - .long 0xbdb0e842,0x8807898b,0x38e7195b,0xdb79c8ee - .long 0x47a17c0a,0xe97c420f,0xc9f8841e,0x00000000 - .long 0x83098086,0x48322bed,0xac1e1170,0x4e6c5a72 - .long 0xfbfd0eff,0x560f8538,0x1e3daed5,0x27362d39 - .long 0x640a0fd9,0x21685ca6,0xd19b5b54,0x3a24362e - .long 0xb10c0a67,0x0f9357e7,0xd2b4ee96,0x9e1b9b91 - .long 0x4f80c0c5,0xa261dc20,0x695a774b,0x161c121a - .long 0x0ae293ba,0xe5c0a02a,0x433c22e0,0x1d121b17 - .long 0x0b0e090d,0xadf28bc7,0xb92db6a8,0xc8141ea9 - .long 0x8557f119,0x4caf7507,0xbbee99dd,0xfda37f60 - .long 0x9ff70126,0xbc5c72f5,0xc544663b,0x345bfb7e - .long 0x768b4329,0xdccb23c6,0x68b6edfc,0x63b8e4f1 - .long 0xcad731dc,0x10426385,0x40139722,0x2084c611 - .long 0x7d854a24,0xf8d2bb3d,0x11aef932,0x6dc729a1 - .long 0x4b1d9e2f,0xf3dcb230,0xec0d8652,0xd077c1e3 - .long 0x6c2bb316,0x99a970b9,0xfa119448,0x2247e964 - .long 0xc4a8fc8c,0x1aa0f03f,0xd8567d2c,0xef223390 - .long 0xc787494e,0xc1d938d1,0xfe8ccaa2,0x3698d40b - .long 0xcfa6f581,0x28a57ade,0x26dab78e,0xa43fadbf - .long 0xe42c3a9d,0x0d507892,0x9b6a5fcc,0x62547e46 - .long 0xc2f68d13,0xe890d8b8,0x5e2e39f7,0xf582c3af - .long 0xbe9f5d80,0x7c69d093,0xa96fd52d,0xb3cf2512 - .long 0x3bc8ac99,0xa710187d,0x6ee89c63,0x7bdb3bbb - .long 0x09cd2678,0xf46e5918,0x01ec9ab7,0xa8834f9a - .long 0x65e6956e,0x7eaaffe6,0x0821bccf,0xe6ef15e8 - .long 0xd9bae79b,0xce4a6f36,0xd4ea9f09,0xd629b07c - .long 0xaf31a4b2,0x312a3f23,0x30c6a594,0xc035a266 - .long 0x37744ebc,0xa6fc82ca,0xb0e090d0,0x1533a7d8 - .long 0x4af10498,0xf741ecda,0x0e7fcd50,0x2f1791f6 - .long 0x8d764dd6,0x4d43efb0,0x54ccaa4d,0xdfe49604 - .long 0xe39ed1b5,0x1b4c6a88,0xb8c12c1f,0x7f466551 - .long 0x049d5eea,0x5d018c35,0x73fa8774,0x2efb0b41 - .long 0x5ab3671d,0x5292dbd2,0x33e91056,0x136dd647 - .long 0x8c9ad761,0x7a37a10c,0x8e59f814,0x89eb133c - .long 0xeecea927,0x35b761c9,0xede11ce5,0x3c7a47b1 - .long 0x599cd2df,0x3f55f273,0x791814ce,0xbf73c737 - .long 0xea53f7cd,0x5b5ffdaa,0x14df3d6f,0x867844db - .long 0x81caaff3,0x3eb968c4,0x2c382434,0x5fc2a340 - .long 0x72161dc3,0x0cbce225,0x8b283c49,0x41ff0d95 - .long 0x7139a801,0xde080cb3,0x9cd8b4e4,0x906456c1 - .long 0x617bcb84,0x70d532b6,0x74486c5c,0x42d0b857 -#endif /* HAVE_AES_DECRYPT */ -#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AES_CBC) || \ - defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ - defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) -#ifndef __APPLE__ - .text - .type L_AES_ARM32_te_data, %object - .size L_AES_ARM32_te_data, 1024 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 -#else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_ARM32_te_data: - .long 0xa5c66363,0x84f87c7c,0x99ee7777,0x8df67b7b - .long 0x0dfff2f2,0xbdd66b6b,0xb1de6f6f,0x5491c5c5 - .long 0x50603030,0x03020101,0xa9ce6767,0x7d562b2b - .long 0x19e7fefe,0x62b5d7d7,0xe64dabab,0x9aec7676 - .long 0x458fcaca,0x9d1f8282,0x4089c9c9,0x87fa7d7d - .long 0x15effafa,0xebb25959,0xc98e4747,0x0bfbf0f0 - .long 0xec41adad,0x67b3d4d4,0xfd5fa2a2,0xea45afaf - .long 0xbf239c9c,0xf753a4a4,0x96e47272,0x5b9bc0c0 - .long 0xc275b7b7,0x1ce1fdfd,0xae3d9393,0x6a4c2626 - .long 0x5a6c3636,0x417e3f3f,0x02f5f7f7,0x4f83cccc - .long 0x5c683434,0xf451a5a5,0x34d1e5e5,0x08f9f1f1 - .long 0x93e27171,0x73abd8d8,0x53623131,0x3f2a1515 - .long 0x0c080404,0x5295c7c7,0x65462323,0x5e9dc3c3 - .long 0x28301818,0xa1379696,0x0f0a0505,0xb52f9a9a - .long 0x090e0707,0x36241212,0x9b1b8080,0x3ddfe2e2 - .long 0x26cdebeb,0x694e2727,0xcd7fb2b2,0x9fea7575 - .long 0x1b120909,0x9e1d8383,0x74582c2c,0x2e341a1a - .long 0x2d361b1b,0xb2dc6e6e,0xeeb45a5a,0xfb5ba0a0 - .long 0xf6a45252,0x4d763b3b,0x61b7d6d6,0xce7db3b3 - .long 0x7b522929,0x3edde3e3,0x715e2f2f,0x97138484 - .long 0xf5a65353,0x68b9d1d1,0x00000000,0x2cc1eded - .long 0x60402020,0x1fe3fcfc,0xc879b1b1,0xedb65b5b - .long 0xbed46a6a,0x468dcbcb,0xd967bebe,0x4b723939 - .long 0xde944a4a,0xd4984c4c,0xe8b05858,0x4a85cfcf - .long 0x6bbbd0d0,0x2ac5efef,0xe54faaaa,0x16edfbfb - .long 0xc5864343,0xd79a4d4d,0x55663333,0x94118585 - .long 0xcf8a4545,0x10e9f9f9,0x06040202,0x81fe7f7f - .long 0xf0a05050,0x44783c3c,0xba259f9f,0xe34ba8a8 - .long 0xf3a25151,0xfe5da3a3,0xc0804040,0x8a058f8f - .long 0xad3f9292,0xbc219d9d,0x48703838,0x04f1f5f5 - .long 0xdf63bcbc,0xc177b6b6,0x75afdada,0x63422121 - .long 0x30201010,0x1ae5ffff,0x0efdf3f3,0x6dbfd2d2 - .long 0x4c81cdcd,0x14180c0c,0x35261313,0x2fc3ecec - .long 0xe1be5f5f,0xa2359797,0xcc884444,0x392e1717 - .long 0x5793c4c4,0xf255a7a7,0x82fc7e7e,0x477a3d3d - .long 0xacc86464,0xe7ba5d5d,0x2b321919,0x95e67373 - .long 0xa0c06060,0x98198181,0xd19e4f4f,0x7fa3dcdc - .long 0x66442222,0x7e542a2a,0xab3b9090,0x830b8888 - .long 0xca8c4646,0x29c7eeee,0xd36bb8b8,0x3c281414 - .long 0x79a7dede,0xe2bc5e5e,0x1d160b0b,0x76addbdb - .long 0x3bdbe0e0,0x56643232,0x4e743a3a,0x1e140a0a - .long 0xdb924949,0x0a0c0606,0x6c482424,0xe4b85c5c - .long 0x5d9fc2c2,0x6ebdd3d3,0xef43acac,0xa6c46262 - .long 0xa8399191,0xa4319595,0x37d3e4e4,0x8bf27979 - .long 0x32d5e7e7,0x438bc8c8,0x596e3737,0xb7da6d6d - .long 0x8c018d8d,0x64b1d5d5,0xd29c4e4e,0xe049a9a9 - .long 0xb4d86c6c,0xfaac5656,0x07f3f4f4,0x25cfeaea - .long 0xafca6565,0x8ef47a7a,0xe947aeae,0x18100808 - .long 0xd56fbaba,0x88f07878,0x6f4a2525,0x725c2e2e - .long 0x24381c1c,0xf157a6a6,0xc773b4b4,0x5197c6c6 - .long 0x23cbe8e8,0x7ca1dddd,0x9ce87474,0x213e1f1f - .long 0xdd964b4b,0xdc61bdbd,0x860d8b8b,0x850f8a8a - .long 0x90e07070,0x427c3e3e,0xc471b5b5,0xaacc6666 - .long 0xd8904848,0x05060303,0x01f7f6f6,0x121c0e0e - .long 0xa3c26161,0x5f6a3535,0xf9ae5757,0xd069b9b9 - .long 0x91178686,0x5899c1c1,0x273a1d1d,0xb9279e9e - .long 0x38d9e1e1,0x13ebf8f8,0xb32b9898,0x33221111 - .long 0xbbd26969,0x70a9d9d9,0x89078e8e,0xa7339494 - .long 0xb62d9b9b,0x223c1e1e,0x92158787,0x20c9e9e9 - .long 0x4987cece,0xffaa5555,0x78502828,0x7aa5dfdf - .long 0x8f038c8c,0xf859a1a1,0x80098989,0x171a0d0d - .long 0xda65bfbf,0x31d7e6e6,0xc6844242,0xb8d06868 - .long 0xc3824141,0xb0299999,0x775a2d2d,0x111e0f0f - .long 0xcb7bb0b0,0xfca85454,0xd66dbbbb,0x3a2c1616 -#endif /* HAVE_AES_DECRYPT || HAVE_AES_CBC || HAVE_AESCCM || HAVE_AESGCM || - * WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */ -#ifdef HAVE_AES_DECRYPT -#ifndef __APPLE__ - .text - .type L_AES_ARM32_td, %object - .size L_AES_ARM32_td, 12 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 -#else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_ARM32_td: - .long L_AES_ARM32_td_data -#endif /* HAVE_AES_DECRYPT */ -#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AES_CBC) || \ - defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ - defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) -#ifndef __APPLE__ - .text - .type L_AES_ARM32_te, %object - .size L_AES_ARM32_te, 12 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 -#else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_ARM32_te: - .long L_AES_ARM32_te_data -#endif /* HAVE_AES_DECRYPT || HAVE_AES_CBC || HAVE_AESCCM || HAVE_AESGCM || - * WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */ -#ifdef HAVE_AES_DECRYPT - .text - .align 4 - .globl AES_invert_key - .type AES_invert_key, %function -AES_invert_key: - push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - adr r12, L_AES_ARM32_te - ldr r12, [r12] - adr lr, L_AES_ARM32_td - ldr lr, [lr] - add r10, r0, r1, lsl #4 - mov r11, r1 -L_AES_invert_key_loop: - ldm r0, {r2, r3, r4, r5} - ldm r10, {r6, r7, r8, r9} - stm r10, {r2, r3, r4, r5} - stm r0!, {r6, r7, r8, r9} - subs r11, r11, #2 - sub r10, r10, #16 - bne L_AES_invert_key_loop - sub r0, r0, r1, lsl #3 - add r0, r0, #16 - sub r11, r1, #1 -L_AES_invert_key_mix_loop: - ldm r0, {r2, r3, r4, r5} -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r2, #24 - lsr r6, r6, #24 -#else - uxtb r6, r2 -#endif -#else - ubfx r6, r2, #0, #8 -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r2, #16 - lsr r7, r7, #24 -#else - uxtb r7, r2, ror #8 -#endif -#else - ubfx r7, r2, #8, #8 -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r2, #8 - lsr r8, r8, #24 -#else - uxtb r8, r2, ror #16 -#endif -#else - ubfx r8, r2, #16, #8 -#endif - lsr r9, r2, #24 - ldrb r6, [r12, r6, lsl #2] - ldrb r7, [r12, r7, lsl #2] - ldrb r8, [r12, r8, lsl #2] - ldrb r9, [r12, r9, lsl #2] - ldr r6, [lr, r6, lsl #2] - ldr r7, [lr, r7, lsl #2] - ldr r8, [lr, r8, lsl #2] - ldr r9, [lr, r9, lsl #2] - eor r8, r8, r6, ror #16 - eor r8, r8, r7, ror #8 - eor r8, r8, r9, ror #24 - str r8, [r0], #4 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r3, #24 - lsr r6, r6, #24 -#else - uxtb r6, r3 + uxtb r5, r10, ror #16 #endif #else - ubfx r6, r3, #0, #8 + ubfx r5, r10, #16, #8 #endif + eor r4, r4, r7, ror #24 + lsr r7, r9, #24 + eor r4, r4, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r3, #16 - lsr r7, r7, #24 + lsl lr, r11, #16 + lsr lr, lr, #24 #else - uxtb r7, r3, ror #8 + uxtb lr, r11, ror #8 #endif #else - ubfx r7, r3, #8, #8 + ubfx lr, r11, #8, #8 #endif + eor r4, r4, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r3, #8 - lsr r8, r8, #24 + lsl r2, r8, #24 + lsr r2, r2, #24 #else - uxtb r8, r3, ror #16 + uxtb r2, r8 #endif #else - ubfx r8, r3, #16, #8 + ubfx r2, r8, #0, #8 #endif - lsr r9, r3, #24 - ldrb r6, [r12, r6, lsl #2] - ldrb r7, [r12, r7, lsl #2] - ldrb r8, [r12, r8, lsl #2] - ldrb r9, [r12, r9, lsl #2] - ldr r6, [lr, r6, lsl #2] - ldr r7, [lr, r7, lsl #2] - ldr r8, [lr, r8, lsl #2] - ldr r9, [lr, r9, lsl #2] - eor r8, r8, r6, ror #16 - eor r8, r8, r7, ror #8 - eor r8, r8, r9, ror #24 - str r8, [r0], #4 + ldr r5, [r0, r5, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r4, #24 + lsl r6, r11, #8 lsr r6, r6, #24 #else - uxtb r6, r4 + uxtb r6, r11, ror #16 #endif #else - ubfx r6, r4, #0, #8 + ubfx r6, r11, #16, #8 #endif + eor r5, r5, r7, ror #24 + lsr r7, r10, #24 + eor r5, r5, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r4, #16 - lsr r7, r7, #24 + lsl lr, r8, #16 + lsr lr, lr, #24 #else - uxtb r7, r4, ror #8 + uxtb lr, r8, ror #8 #endif #else - ubfx r7, r4, #8, #8 + ubfx lr, r8, #8, #8 #endif + eor r5, r5, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r4, #8 - lsr r8, r8, #24 + lsl r2, r9, #24 + lsr r2, r2, #24 #else - uxtb r8, r4, ror #16 + uxtb r2, r9 #endif #else - ubfx r8, r4, #16, #8 + ubfx r2, r9, #0, #8 #endif - lsr r9, r4, #24 - ldrb r6, [r12, r6, lsl #2] - ldrb r7, [r12, r7, lsl #2] - ldrb r8, [r12, r8, lsl #2] - ldrb r9, [r12, r9, lsl #2] - ldr r6, [lr, r6, lsl #2] - ldr r7, [lr, r7, lsl #2] - ldr r8, [lr, r8, lsl #2] - ldr r9, [lr, r9, lsl #2] - eor r8, r8, r6, ror #16 - eor r8, r8, r7, ror #8 - eor r8, r8, r9, ror #24 - str r8, [r0], #4 + ldr r6, [r0, r6, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r5, #24 - lsr r6, r6, #24 + lsl r10, r10, #24 + lsr r10, r10, #24 #else - uxtb r6, r5 + uxtb r10, r10 #endif #else - ubfx r6, r5, #0, #8 + ubfx r10, r10, #0, #8 #endif + eor r6, r6, r7, ror #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r5, #16 + lsl r7, r8, #8 lsr r7, r7, #24 #else - uxtb r7, r5, ror #8 + uxtb r7, r8, ror #16 #endif #else - ubfx r7, r5, #8, #8 + ubfx r7, r8, #16, #8 #endif + eor r6, r6, lr, ror #8 + lsr lr, r11, #24 + eor r6, r6, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r5, #8 - lsr r8, r8, #24 + lsl r2, r9, #16 + lsr r2, r2, #24 #else - uxtb r8, r5, ror #16 + uxtb r2, r9, ror #8 #endif #else - ubfx r8, r5, #16, #8 + ubfx r2, r9, #8, #8 #endif - lsr r9, r5, #24 - ldrb r6, [r12, r6, lsl #2] - ldrb r7, [r12, r7, lsl #2] - ldrb r8, [r12, r8, lsl #2] - ldrb r9, [r12, r9, lsl #2] - ldr r6, [lr, r6, lsl #2] - ldr r7, [lr, r7, lsl #2] - ldr r8, [lr, r8, lsl #2] - ldr r9, [lr, r9, lsl #2] - eor r8, r8, r6, ror #16 - eor r8, r8, r7, ror #8 - eor r8, r8, r9, ror #24 - str r8, [r0], #4 - subs r11, r11, #1 - bne L_AES_invert_key_mix_loop - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size AES_invert_key,.-AES_invert_key -#endif /* HAVE_AES_DECRYPT */ -#ifndef __APPLE__ - .text - .type L_AES_ARM32_rcon, %object - .size L_AES_ARM32_rcon, 40 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 -#else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_ARM32_rcon: - .long 0x01000000,0x02000000,0x04000000,0x08000000 - .long 0x10000000,0x20000000,0x40000000,0x80000000 - .long 0x1b000000,0x36000000 - .text - .align 4 - .globl AES_set_encrypt_key - .type AES_set_encrypt_key, %function -AES_set_encrypt_key: - push {r4, r5, r6, r7, r8, lr} - adr r8, L_AES_ARM32_te - ldr r8, [r8] - adr lr, L_AES_ARM32_rcon - cmp r1, #0x80 - beq L_AES_set_encrypt_key_start_128 - cmp r1, #0xc0 - beq L_AES_set_encrypt_key_start_192 - ldr r4, [r0] - ldr r5, [r0, #4] - ldr r6, [r0, #8] - ldr r7, [r0, #12] -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - # REV r4, r4 - eor r3, r4, r4, ror #16 - bic r3, r3, #0xff0000 - ror r4, r4, #8 - eor r4, r4, r3, lsr #8 - # REV r5, r5 - eor r3, r5, r5, ror #16 - bic r3, r3, #0xff0000 - ror r5, r5, #8 - eor r5, r5, r3, lsr #8 - # REV r6, r6 - eor r3, r6, r6, ror #16 - bic r3, r3, #0xff0000 - ror r6, r6, #8 - eor r6, r6, r3, lsr #8 - # REV r7, r7 - eor r3, r7, r7, ror #16 - bic r3, r3, #0xff0000 - ror r7, r7, #8 - eor r7, r7, r3, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - stm r2!, {r4, r5, r6, r7} - ldr r4, [r0, #16] - ldr r5, [r0, #20] - ldr r6, [r0, #24] - ldr r7, [r0, #28] + ldr r10, [r0, r10, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr r2, [r0, r2, lsl #2] + eor lr, lr, r10, ror #24 + ldm r3!, {r8, r9, r10, r11} + eor r7, r7, lr, ror #24 + eor r7, r7, r2, ror #8 + # XOR in Key Schedule + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + subs r1, r1, #1 + bne L_AES_encrypt_block_nr +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - # REV r4, r4 - eor r3, r4, r4, ror #16 - bic r3, r3, #0xff0000 - ror r4, r4, #8 - eor r4, r4, r3, lsr #8 - # REV r5, r5 - eor r3, r5, r5, ror #16 - bic r3, r3, #0xff0000 - ror r5, r5, #8 - eor r5, r5, r3, lsr #8 - # REV r6, r6 - eor r3, r6, r6, ror #16 - bic r3, r3, #0xff0000 - ror r6, r6, #8 - eor r6, r6, r3, lsr #8 - # REV r7, r7 - eor r3, r7, r7, ror #16 - bic r3, r3, #0xff0000 - ror r7, r7, #8 - eor r7, r7, r3, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - stm r2, {r4, r5, r6, r7} - sub r2, r2, #16 - mov r12, #6 -L_AES_set_encrypt_key_loop_256: + lsl r8, r5, #8 + lsr r8, r8, #24 +#else + uxtb r8, r5, ror #16 +#endif +#else + ubfx r8, r5, #16, #8 +#endif + lsr r11, r4, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r7, #24 - lsr r4, r4, #24 + lsl lr, r6, #16 + lsr lr, lr, #24 #else - uxtb r4, r7 + uxtb lr, r6, ror #8 #endif #else - ubfx r4, r7, #0, #8 + ubfx lr, r6, #8, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r7, #16 - lsr r5, r5, #24 + lsl r2, r7, #24 + lsr r2, r2, #24 #else - uxtb r5, r7, ror #8 + uxtb r2, r7 #endif #else - ubfx r5, r7, #8, #8 + ubfx r2, r7, #0, #8 #endif + ldr r8, [r0, r8, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r7, #8 - lsr r6, r6, #24 + lsl r9, r6, #8 + lsr r9, r9, #24 #else - uxtb r6, r7, ror #16 + uxtb r9, r6, ror #16 #endif #else - ubfx r6, r7, #16, #8 + ubfx r9, r6, #16, #8 #endif - lsr r7, r7, #24 - ldrb r4, [r8, r4, lsl #2] - ldrb r5, [r8, r5, lsl #2] - ldrb r6, [r8, r6, lsl #2] - ldrb r7, [r8, r7, lsl #2] - eor r3, r7, r4, lsl #8 - eor r3, r3, r5, lsl #16 - eor r3, r3, r6, lsl #24 - ldm r2!, {r4, r5, r6, r7} - eor r4, r4, r3 - ldm lr!, {r3} - eor r4, r4, r3 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - add r2, r2, #16 - stm r2, {r4, r5, r6, r7} - sub r2, r2, #16 + eor r8, r8, r11, ror #24 + lsr r11, r5, #24 + eor r8, r8, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r7, #16 - lsr r4, r4, #24 + lsl lr, r7, #16 + lsr lr, lr, #24 #else - uxtb r4, r7, ror #8 + uxtb lr, r7, ror #8 #endif #else - ubfx r4, r7, #8, #8 + ubfx lr, r7, #8, #8 #endif + eor r8, r8, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r7, #8 - lsr r5, r5, #24 + lsl r2, r4, #24 + lsr r2, r2, #24 #else - uxtb r5, r7, ror #16 + uxtb r2, r4 #endif #else - ubfx r5, r7, #16, #8 + ubfx r2, r4, #0, #8 #endif - lsr r6, r7, #24 + ldr r9, [r0, r9, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r3, r7, #24 - lsr r3, r3, #24 + lsl r10, r7, #8 + lsr r10, r10, #24 #else - uxtb r3, r7 + uxtb r10, r7, ror #16 #endif #else - ubfx r3, r7, #0, #8 + ubfx r10, r7, #16, #8 #endif - ldrb r4, [r8, r4, lsl #2] - ldrb r6, [r8, r6, lsl #2] - ldrb r5, [r8, r5, lsl #2] - ldrb r3, [r8, r3, lsl #2] - eor r3, r3, r4, lsl #8 - eor r3, r3, r5, lsl #16 - eor r3, r3, r6, lsl #24 - ldm r2!, {r4, r5, r6, r7} - eor r4, r4, r3 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - add r2, r2, #16 - stm r2, {r4, r5, r6, r7} - sub r2, r2, #16 - subs r12, r12, #1 - bne L_AES_set_encrypt_key_loop_256 + eor r9, r9, r11, ror #24 + lsr r11, r6, #24 + eor r9, r9, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r7, #24 - lsr r4, r4, #24 + lsl lr, r4, #16 + lsr lr, lr, #24 #else - uxtb r4, r7 + uxtb lr, r4, ror #8 #endif #else - ubfx r4, r7, #0, #8 + ubfx lr, r4, #8, #8 #endif + eor r9, r9, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r7, #16 - lsr r5, r5, #24 + lsl r2, r5, #24 + lsr r2, r2, #24 #else - uxtb r5, r7, ror #8 + uxtb r2, r5 #endif #else - ubfx r5, r7, #8, #8 + ubfx r2, r5, #0, #8 #endif + ldr r10, [r0, r10, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r7, #8 + lsl r6, r6, #24 lsr r6, r6, #24 #else - uxtb r6, r7, ror #16 + uxtb r6, r6 #endif #else - ubfx r6, r7, #16, #8 + ubfx r6, r6, #0, #8 #endif - lsr r7, r7, #24 - ldrb r4, [r8, r4, lsl #2] - ldrb r5, [r8, r5, lsl #2] - ldrb r6, [r8, r6, lsl #2] - ldrb r7, [r8, r7, lsl #2] - eor r3, r7, r4, lsl #8 - eor r3, r3, r5, lsl #16 - eor r3, r3, r6, lsl #24 - ldm r2!, {r4, r5, r6, r7} - eor r4, r4, r3 - ldm lr!, {r3} - eor r4, r4, r3 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - add r2, r2, #16 - stm r2, {r4, r5, r6, r7} - sub r2, r2, #16 - b L_AES_set_encrypt_key_end -L_AES_set_encrypt_key_start_192: - ldr r4, [r0] - ldr r5, [r0, #4] - ldr r6, [r0, #8] - ldr r7, [r0, #12] - ldr r1, [r0, #20] - ldr r0, [r0, #16] + eor r10, r10, r11, ror #24 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - # REV r4, r4 - eor r3, r4, r4, ror #16 - bic r3, r3, #0xff0000 - ror r4, r4, #8 - eor r4, r4, r3, lsr #8 - # REV r5, r5 - eor r3, r5, r5, ror #16 - bic r3, r3, #0xff0000 - ror r5, r5, #8 - eor r5, r5, r3, lsr #8 - # REV r6, r6 - eor r3, r6, r6, ror #16 - bic r3, r3, #0xff0000 - ror r6, r6, #8 - eor r6, r6, r3, lsr #8 - # REV r7, r7 - eor r3, r7, r7, ror #16 - bic r3, r3, #0xff0000 - ror r7, r7, #8 - eor r7, r7, r3, lsr #8 - # REV r0, r0 - eor r3, r0, r0, ror #16 - bic r3, r3, #0xff0000 - ror r0, r0, #8 - eor r0, r0, r3, lsr #8 - # REV r1, r1 - eor r3, r1, r1, ror #16 - bic r3, r3, #0xff0000 - ror r1, r1, #8 - eor r1, r1, r3, lsr #8 + lsl r11, r4, #8 + lsr r11, r11, #24 #else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 - rev r0, r0 - rev r1, r1 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - stm r2, {r4, r5, r6, r7} + uxtb r11, r4, ror #16 +#endif +#else + ubfx r11, r4, #16, #8 +#endif + eor r10, r10, lr, ror #8 + lsr lr, r7, #24 + eor r10, r10, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - str r0, [r2, #16] - str r1, [r2, #20] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r5, #16 + lsr r2, r2, #24 #else - strd r0, r1, [r2, #16] + uxtb r2, r5, ror #8 +#endif +#else + ubfx r2, r5, #8, #8 +#endif + ldr r6, [r0, r6, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr r2, [r0, r2, lsl #2] + eor lr, lr, r6, ror #24 + ldm r3!, {r4, r5, r6, r7} + eor r11, r11, lr, ror #24 + eor r11, r11, r2, ror #8 + # XOR in Key Schedule + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r4, r11, #24 + lsr r4, r4, #24 +#else + uxtb r4, r11 +#endif +#else + ubfx r4, r11, #0, #8 +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r10, #16 + lsr r7, r7, #24 +#else + uxtb r7, r10, ror #8 +#endif +#else + ubfx r7, r10, #8, #8 #endif - mov r7, r1 - mov r12, #7 -L_AES_set_encrypt_key_loop_192: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r0, r7, #24 - lsr r0, r0, #24 + lsl lr, r9, #8 + lsr lr, lr, #24 #else - uxtb r0, r7 + uxtb lr, r9, ror #16 #endif #else - ubfx r0, r7, #0, #8 + ubfx lr, r9, #16, #8 #endif + lsr r2, r8, #24 + ldrb r4, [r0, r4, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r1, r7, #16 - lsr r1, r1, #24 + lsl r5, r8, #24 + lsr r5, r5, #24 #else - uxtb r1, r7, ror #8 + uxtb r5, r8 #endif #else - ubfx r1, r7, #8, #8 + ubfx r5, r8, #0, #8 #endif + eor r4, r4, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r7, #8 - lsr r4, r4, #24 + lsl r7, r11, #16 + lsr r7, r7, #24 #else - uxtb r4, r7, ror #16 + uxtb r7, r11, ror #8 #endif #else - ubfx r4, r7, #16, #8 + ubfx r7, r11, #8, #8 #endif - lsr r7, r7, #24 - ldrb r0, [r8, r0, lsl #2] - ldrb r1, [r8, r1, lsl #2] - ldrb r4, [r8, r4, lsl #2] - ldrb r7, [r8, r7, lsl #2] - eor r3, r7, r0, lsl #8 - eor r3, r3, r1, lsl #16 - eor r3, r3, r4, lsl #24 - ldm r2!, {r0, r1, r4, r5, r6, r7} - eor r0, r0, r3 - ldm lr!, {r3} - eor r0, r0, r3 - eor r1, r1, r0 - eor r4, r4, r1 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - stm r2, {r0, r1, r4, r5, r6, r7} - subs r12, r12, #1 - bne L_AES_set_encrypt_key_loop_192 + eor r4, r4, lr, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r0, r7, #24 - lsr r0, r0, #24 + lsl lr, r10, #8 + lsr lr, lr, #24 #else - uxtb r0, r7 + uxtb lr, r10, ror #16 #endif #else - ubfx r0, r7, #0, #8 + ubfx lr, r10, #16, #8 #endif + eor r4, r4, r2, lsl #24 + lsr r2, r9, #24 + ldrb r5, [r0, r5, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r1, r7, #16 - lsr r1, r1, #24 + lsl r6, r9, #24 + lsr r6, r6, #24 #else - uxtb r1, r7, ror #8 + uxtb r6, r9 #endif #else - ubfx r1, r7, #8, #8 + ubfx r6, r9, #0, #8 #endif + eor r5, r5, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r7, #8 - lsr r4, r4, #24 + lsl r7, r8, #16 + lsr r7, r7, #24 #else - uxtb r4, r7, ror #16 + uxtb r7, r8, ror #8 #endif #else - ubfx r4, r7, #16, #8 + ubfx r7, r8, #8, #8 #endif - lsr r7, r7, #24 - ldrb r0, [r8, r0, lsl #2] - ldrb r1, [r8, r1, lsl #2] - ldrb r4, [r8, r4, lsl #2] - ldrb r7, [r8, r7, lsl #2] - eor r3, r7, r0, lsl #8 - eor r3, r3, r1, lsl #16 - eor r3, r3, r4, lsl #24 - ldm r2!, {r0, r1, r4, r5, r6, r7} - eor r0, r0, r3 - ldm lr!, {r3} - eor r0, r0, r3 - eor r1, r1, r0 - eor r4, r4, r1 - eor r5, r5, r4 - stm r2, {r0, r1, r4, r5} - b L_AES_set_encrypt_key_end -L_AES_set_encrypt_key_start_128: - ldr r4, [r0] - ldr r5, [r0, #4] - ldr r6, [r0, #8] - ldr r7, [r0, #12] + eor r5, r5, lr, lsl #16 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - # REV r4, r4 - eor r3, r4, r4, ror #16 - bic r3, r3, #0xff0000 - ror r4, r4, #8 - eor r4, r4, r3, lsr #8 - # REV r5, r5 - eor r3, r5, r5, ror #16 - bic r3, r3, #0xff0000 - ror r5, r5, #8 - eor r5, r5, r3, lsr #8 - # REV r6, r6 - eor r3, r6, r6, ror #16 - bic r3, r3, #0xff0000 - ror r6, r6, #8 - eor r6, r6, r3, lsr #8 - # REV r7, r7 - eor r3, r7, r7, ror #16 - bic r3, r3, #0xff0000 - ror r7, r7, #8 - eor r7, r7, r3, lsr #8 + lsl lr, r11, #8 + lsr lr, lr, #24 #else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - stm r2, {r4, r5, r6, r7} - mov r12, #10 -L_AES_set_encrypt_key_loop_128: + uxtb lr, r11, ror #16 +#endif +#else + ubfx lr, r11, #16, #8 +#endif + eor r5, r5, r2, lsl #24 + lsr r2, r10, #24 + ldrb r6, [r0, r6, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] + lsr r11, r11, #24 + eor r6, r6, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r7, #24 - lsr r4, r4, #24 + lsl r7, r10, #24 + lsr r7, r7, #24 #else - uxtb r4, r7 + uxtb r7, r10 #endif #else - ubfx r4, r7, #0, #8 + ubfx r7, r10, #0, #8 #endif + eor r6, r6, lr, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r7, #16 - lsr r5, r5, #24 + lsl lr, r9, #16 + lsr lr, lr, #24 #else - uxtb r5, r7, ror #8 + uxtb lr, r9, ror #8 #endif #else - ubfx r5, r7, #8, #8 + ubfx lr, r9, #8, #8 #endif + eor r6, r6, r2, lsl #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r7, #8 - lsr r6, r6, #24 + lsl r2, r8, #8 + lsr r2, r2, #24 #else - uxtb r6, r7, ror #16 + uxtb r2, r8, ror #16 #endif #else - ubfx r6, r7, #16, #8 + ubfx r2, r8, #16, #8 #endif - lsr r7, r7, #24 - ldrb r4, [r8, r4, lsl #2] - ldrb r5, [r8, r5, lsl #2] - ldrb r6, [r8, r6, lsl #2] - ldrb r7, [r8, r7, lsl #2] - eor r3, r7, r4, lsl #8 - eor r3, r3, r5, lsl #16 - eor r3, r3, r6, lsl #24 - ldm r2!, {r4, r5, r6, r7} - eor r4, r4, r3 - ldm lr!, {r3} - eor r4, r4, r3 - eor r5, r5, r4 - eor r6, r6, r5 - eor r7, r7, r6 - stm r2, {r4, r5, r6, r7} - subs r12, r12, #1 - bne L_AES_set_encrypt_key_loop_128 -L_AES_set_encrypt_key_end: - pop {r4, r5, r6, r7, r8, pc} - .size AES_set_encrypt_key,.-AES_set_encrypt_key + ldrb r11, [r0, r11, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] + eor lr, lr, r11, lsl #16 + ldm r3, {r8, r9, r10, r11} + eor r7, r7, lr, lsl #8 + eor r7, r7, r2, lsl #16 + # XOR in Key Schedule + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + pop {pc} + .size AES_encrypt_block,.-AES_encrypt_block +#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ +#if defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ + defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || \ + defined(HAVE_AES_ECB) +#ifndef __APPLE__ + .text + .type L_AES_ARM32_te_ecb, %object + .size L_AES_ARM32_te_ecb, 12 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_AES_ARM32_te_ecb: + .long L_AES_ARM32_te_data + .text + .align 4 + .globl AES_ECB_encrypt + .type AES_ECB_encrypt, %function +AES_ECB_encrypt: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + mov lr, r0 + adr r0, L_AES_ARM32_te_ecb + ldr r0, [r0] + ldr r12, [sp, #36] + push {r3} + cmp r12, #10 + beq L_AES_ECB_encrypt_start_block_128 + cmp r12, #12 + beq L_AES_ECB_encrypt_start_block_192 +L_AES_ECB_encrypt_loop_block_256: + ldr r4, [lr] + ldr r5, [lr, #4] + ldr r6, [lr, #8] + ldr r7, [lr, #12] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + eor r8, r4, r4, ror #16 + eor r9, r5, r5, ror #16 + eor r10, r6, r6, ror #16 + eor r11, r7, r7, ror #16 + bic r8, r8, #0xff0000 + bic r9, r9, #0xff0000 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 + ror r6, r6, #8 + ror r7, r7, #8 + eor r4, r4, r8, lsr #8 + eor r5, r5, r9, lsr #8 + eor r6, r6, r10, lsr #8 + eor r7, r7, r11, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + push {r1, r2, lr} + ldm r3!, {r8, r9, r10, r11} + # Round: 0 - XOR in key schedule + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + mov r1, #6 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE - .text - .align 4 - .globl AES_encrypt_block - .type AES_encrypt_block, %function -AES_encrypt_block: - push {lr} -L_AES_encrypt_block_nr: + bl AES_encrypt_block +#else +L_AES_ECB_encrypt_block_nr_256: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -9347,7 +1951,7 @@ L_AES_encrypt_block_nr: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_encrypt_block_nr + bne L_AES_ECB_encrypt_block_nr_256 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -9664,43 +2268,43 @@ L_AES_encrypt_block_nr: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - pop {pc} - .size AES_encrypt_block,.-AES_encrypt_block #endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ -#if defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ - defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || \ - defined(HAVE_AES_ECB) -#ifndef __APPLE__ - .text - .type L_AES_ARM32_te_ecb, %object - .size L_AES_ARM32_te_ecb, 12 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 + pop {r1, r2, lr} + ldr r3, [sp] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + eor r8, r4, r4, ror #16 + eor r9, r5, r5, ror #16 + eor r10, r6, r6, ror #16 + eor r11, r7, r7, ror #16 + bic r8, r8, #0xff0000 + bic r9, r9, #0xff0000 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 + ror r6, r6, #8 + ror r7, r7, #8 + eor r4, r4, r8, lsr #8 + eor r5, r5, r9, lsr #8 + eor r6, r6, r10, lsr #8 + eor r7, r7, r11, lsr #8 #else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_ARM32_te_ecb: - .long L_AES_ARM32_te_data - .text - .align 4 - .globl AES_ECB_encrypt - .type AES_ECB_encrypt, %function -AES_ECB_encrypt: - push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - mov lr, r0 - adr r0, L_AES_ARM32_te_ecb - ldr r0, [r0] - ldr r12, [sp, #36] - push {r3} - cmp r12, #10 - beq L_AES_ECB_encrypt_start_block_128 - cmp r12, #12 - beq L_AES_ECB_encrypt_start_block_192 -L_AES_ECB_encrypt_loop_block_256: + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + str r4, [r1] + str r5, [r1, #4] + str r6, [r1, #8] + str r7, [r1, #12] + subs r2, r2, #16 + add lr, lr, #16 + add r1, r1, #16 + bne L_AES_ECB_encrypt_loop_block_256 + b L_AES_ECB_encrypt_end +L_AES_ECB_encrypt_start_block_192: +L_AES_ECB_encrypt_loop_block_192: ldr r4, [lr] ldr r5, [lr, #4] ldr r6, [lr, #8] @@ -9735,11 +2339,11 @@ L_AES_ECB_encrypt_loop_block_256: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #6 + mov r1, #5 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_ECB_encrypt_block_nr_256: +L_AES_ECB_encrypt_block_nr_192: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -10057,7 +2661,7 @@ L_AES_ECB_encrypt_block_nr_256: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_ECB_encrypt_block_nr_256 + bne L_AES_ECB_encrypt_block_nr_192 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -10407,10 +3011,10 @@ L_AES_ECB_encrypt_block_nr_256: subs r2, r2, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_ECB_encrypt_loop_block_256 + bne L_AES_ECB_encrypt_loop_block_192 b L_AES_ECB_encrypt_end -L_AES_ECB_encrypt_start_block_192: -L_AES_ECB_encrypt_loop_block_192: +L_AES_ECB_encrypt_start_block_128: +L_AES_ECB_encrypt_loop_block_128: ldr r4, [lr] ldr r5, [lr, #4] ldr r6, [lr, #8] @@ -10445,11 +3049,11 @@ L_AES_ECB_encrypt_loop_block_192: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #5 + mov r1, #4 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_ECB_encrypt_block_nr_192: +L_AES_ECB_encrypt_block_nr_128: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -10767,7 +3371,7 @@ L_AES_ECB_encrypt_block_nr_192: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_ECB_encrypt_block_nr_192 + bne L_AES_ECB_encrypt_block_nr_128 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -11117,14 +3721,56 @@ L_AES_ECB_encrypt_block_nr_192: subs r2, r2, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_ECB_encrypt_loop_block_192 - b L_AES_ECB_encrypt_end -L_AES_ECB_encrypt_start_block_128: -L_AES_ECB_encrypt_loop_block_128: - ldr r4, [lr] - ldr r5, [lr, #4] - ldr r6, [lr, #8] - ldr r7, [lr, #12] + bne L_AES_ECB_encrypt_loop_block_128 +L_AES_ECB_encrypt_end: + pop {r3} + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size AES_ECB_encrypt,.-AES_ECB_encrypt +#endif /* HAVE_AESCCM || HAVE_AESGCM || WOLFSSL_AES_DIRECT || + * WOLFSSL_AES_COUNTER || HAVE_AES_ECB */ +#ifdef HAVE_AES_CBC +#ifndef __APPLE__ + .text + .type L_AES_ARM32_te_cbc, %object + .size L_AES_ARM32_te_cbc, 12 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_AES_ARM32_te_cbc: + .long L_AES_ARM32_te_data + .text + .align 4 + .globl AES_CBC_encrypt + .type AES_CBC_encrypt, %function +AES_CBC_encrypt: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + ldr r8, [sp, #36] + ldr r9, [sp, #40] + mov lr, r0 + adr r0, L_AES_ARM32_te_cbc + ldr r0, [r0] + ldm r9, {r4, r5, r6, r7} + push {r3, r9} + cmp r8, #10 + beq L_AES_CBC_encrypt_start_block_128 + cmp r8, #12 + beq L_AES_CBC_encrypt_start_block_192 +L_AES_CBC_encrypt_loop_block_256: + ldr r8, [lr] + ldr r9, [lr, #4] + ldr r10, [lr, #8] + ldr r11, [lr, #12] + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + push {r1, r2, lr} #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) eor r8, r4, r4, ror #16 eor r9, r5, r5, ror #16 @@ -11148,18 +3794,17 @@ L_AES_ECB_encrypt_loop_block_128: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - push {r1, r2, lr} ldm r3!, {r8, r9, r10, r11} # Round: 0 - XOR in key schedule eor r4, r4, r8 eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #4 + mov r1, #6 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_ECB_encrypt_block_nr_128: +L_AES_CBC_encrypt_block_nr_256: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -11477,7 +4122,7 @@ L_AES_ECB_encrypt_block_nr_128: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_ECB_encrypt_block_nr_128 + bne L_AES_CBC_encrypt_block_nr_256 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -11803,71 +4448,34 @@ L_AES_ECB_encrypt_block_nr_128: eor r10, r6, r6, ror #16 eor r11, r7, r7, ror #16 bic r8, r8, #0xff0000 - bic r9, r9, #0xff0000 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - ror r6, r6, #8 - ror r7, r7, #8 - eor r4, r4, r8, lsr #8 - eor r5, r5, r9, lsr #8 - eor r6, r6, r10, lsr #8 - eor r7, r7, r11, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - str r4, [r1] - str r5, [r1, #4] - str r6, [r1, #8] - str r7, [r1, #12] - subs r2, r2, #16 - add lr, lr, #16 - add r1, r1, #16 - bne L_AES_ECB_encrypt_loop_block_128 -L_AES_ECB_encrypt_end: - pop {r3} - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size AES_ECB_encrypt,.-AES_ECB_encrypt -#endif /* HAVE_AESCCM || HAVE_AESGCM || WOLFSSL_AES_DIRECT || - * WOLFSSL_AES_COUNTER || HAVE_AES_ECB */ -#ifdef HAVE_AES_CBC -#ifndef __APPLE__ - .text - .type L_AES_ARM32_te_cbc, %object - .size L_AES_ARM32_te_cbc, 12 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 -#else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_ARM32_te_cbc: - .long L_AES_ARM32_te_data - .text - .align 4 - .globl AES_CBC_encrypt - .type AES_CBC_encrypt, %function -AES_CBC_encrypt: - push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - ldr r8, [sp, #36] - ldr r9, [sp, #40] - mov lr, r0 - adr r0, L_AES_ARM32_te_cbc - ldr r0, [r0] - ldm r9, {r4, r5, r6, r7} - push {r3, r9} - cmp r8, #10 - beq L_AES_CBC_encrypt_start_block_128 - cmp r8, #12 - beq L_AES_CBC_encrypt_start_block_192 -L_AES_CBC_encrypt_loop_block_256: + bic r9, r9, #0xff0000 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 + ror r6, r6, #8 + ror r7, r7, #8 + eor r4, r4, r8, lsr #8 + eor r5, r5, r9, lsr #8 + eor r6, r6, r10, lsr #8 + eor r7, r7, r11, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + str r4, [r1] + str r5, [r1, #4] + str r6, [r1, #8] + str r7, [r1, #12] + subs r2, r2, #16 + add lr, lr, #16 + add r1, r1, #16 + bne L_AES_CBC_encrypt_loop_block_256 + b L_AES_CBC_encrypt_end +L_AES_CBC_encrypt_start_block_192: +L_AES_CBC_encrypt_loop_block_192: ldr r8, [lr] ldr r9, [lr, #4] ldr r10, [lr, #8] @@ -11906,11 +4514,11 @@ L_AES_CBC_encrypt_loop_block_256: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #6 + mov r1, #5 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_CBC_encrypt_block_nr_256: +L_AES_CBC_encrypt_block_nr_192: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -12228,7 +4836,7 @@ L_AES_CBC_encrypt_block_nr_256: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_CBC_encrypt_block_nr_256 + bne L_AES_CBC_encrypt_block_nr_192 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -12578,10 +5186,10 @@ L_AES_CBC_encrypt_block_nr_256: subs r2, r2, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_CBC_encrypt_loop_block_256 + bne L_AES_CBC_encrypt_loop_block_192 b L_AES_CBC_encrypt_end -L_AES_CBC_encrypt_start_block_192: -L_AES_CBC_encrypt_loop_block_192: +L_AES_CBC_encrypt_start_block_128: +L_AES_CBC_encrypt_loop_block_128: ldr r8, [lr] ldr r9, [lr, #4] ldr r10, [lr, #8] @@ -12620,11 +5228,11 @@ L_AES_CBC_encrypt_loop_block_192: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #5 + mov r1, #4 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_CBC_encrypt_block_nr_192: +L_AES_CBC_encrypt_block_nr_128: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -12942,7 +5550,7 @@ L_AES_CBC_encrypt_block_nr_192: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_CBC_encrypt_block_nr_192 + bne L_AES_CBC_encrypt_block_nr_128 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -13292,34 +5900,56 @@ L_AES_CBC_encrypt_block_nr_192: subs r2, r2, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_CBC_encrypt_loop_block_192 - b L_AES_CBC_encrypt_end -L_AES_CBC_encrypt_start_block_128: -L_AES_CBC_encrypt_loop_block_128: - ldr r8, [lr] - ldr r9, [lr, #4] - ldr r10, [lr, #8] - ldr r11, [lr, #12] - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 - push {r1, r2, lr} + bne L_AES_CBC_encrypt_loop_block_128 +L_AES_CBC_encrypt_end: + pop {r3, r9} + stm r9, {r4, r5, r6, r7} + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size AES_CBC_encrypt,.-AES_CBC_encrypt +#endif /* HAVE_AES_CBC */ +#ifdef WOLFSSL_AES_COUNTER +#ifndef __APPLE__ + .text + .type L_AES_ARM32_te_ctr, %object + .size L_AES_ARM32_te_ctr, 12 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_AES_ARM32_te_ctr: + .long L_AES_ARM32_te_data + .text + .align 4 + .globl AES_CTR_encrypt + .type AES_CTR_encrypt, %function +AES_CTR_encrypt: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + ldr r12, [sp, #36] + ldr r8, [sp, #40] + mov lr, r0 + adr r0, L_AES_ARM32_te_ctr + ldr r0, [r0] + ldm r8, {r4, r5, r6, r7} #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r8, r4, r4, ror #16 - eor r9, r5, r5, ror #16 - eor r10, r6, r6, ror #16 - eor r11, r7, r7, ror #16 - bic r8, r8, #0xff0000 - bic r9, r9, #0xff0000 + eor r10, r4, r4, ror #16 + eor r11, r5, r5, ror #16 bic r10, r10, #0xff0000 bic r11, r11, #0xff0000 ror r4, r4, #8 ror r5, r5, #8 + eor r4, r4, r10, lsr #8 + eor r5, r5, r11, lsr #8 + eor r10, r6, r6, ror #16 + eor r11, r7, r7, ror #16 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 ror r6, r6, #8 ror r7, r7, #8 - eor r4, r4, r8, lsr #8 - eor r5, r5, r9, lsr #8 eor r6, r6, r10, lsr #8 eor r7, r7, r11, lsr #8 #else @@ -13328,17 +5958,31 @@ L_AES_CBC_encrypt_loop_block_128: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r8, {r4, r5, r6, r7} + push {r3, r8} + cmp r12, #10 + beq L_AES_CTR_encrypt_start_block_128 + cmp r12, #12 + beq L_AES_CTR_encrypt_start_block_192 +L_AES_CTR_encrypt_loop_block_256: + push {r1, r2, lr} + ldr lr, [sp, #16] + adds r11, r7, #1 + adcs r10, r6, #0 + adcs r9, r5, #0 + adc r8, r4, #0 + stm lr, {r8, r9, r10, r11} ldm r3!, {r8, r9, r10, r11} # Round: 0 - XOR in key schedule eor r4, r4, r8 eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #4 + mov r1, #6 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_CBC_encrypt_block_nr_128: +L_AES_CTR_encrypt_block_nr_256: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -13656,7 +6300,7 @@ L_AES_CBC_encrypt_block_nr_128: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_CBC_encrypt_block_nr_128 + bne L_AES_CTR_encrypt_block_nr_256 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -13999,78 +6643,27 @@ L_AES_CBC_encrypt_block_nr_128: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + ldr r8, [lr] + ldr r9, [lr, #4] + ldr r10, [lr, #8] + ldr r11, [lr, #12] + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + ldr r8, [sp, #4] str r4, [r1] str r5, [r1, #4] str r6, [r1, #8] str r7, [r1, #12] + ldm r8, {r4, r5, r6, r7} subs r2, r2, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_CBC_encrypt_loop_block_128 -L_AES_CBC_encrypt_end: - pop {r3, r9} - stm r9, {r4, r5, r6, r7} - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size AES_CBC_encrypt,.-AES_CBC_encrypt -#endif /* HAVE_AES_CBC */ -#ifdef WOLFSSL_AES_COUNTER -#ifndef __APPLE__ - .text - .type L_AES_ARM32_te_ctr, %object - .size L_AES_ARM32_te_ctr, 12 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 -#else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_ARM32_te_ctr: - .long L_AES_ARM32_te_data - .text - .align 4 - .globl AES_CTR_encrypt - .type AES_CTR_encrypt, %function -AES_CTR_encrypt: - push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - ldr r12, [sp, #36] - ldr r8, [sp, #40] - mov lr, r0 - adr r0, L_AES_ARM32_te_ctr - ldr r0, [r0] - ldm r8, {r4, r5, r6, r7} -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r10, r4, r4, ror #16 - eor r11, r5, r5, ror #16 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - eor r4, r4, r10, lsr #8 - eor r5, r5, r11, lsr #8 - eor r10, r6, r6, ror #16 - eor r11, r7, r7, ror #16 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r6, r6, #8 - ror r7, r7, #8 - eor r6, r6, r10, lsr #8 - eor r7, r7, r11, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - stm r8, {r4, r5, r6, r7} - push {r3, r8} - cmp r12, #10 - beq L_AES_CTR_encrypt_start_block_128 - cmp r12, #12 - beq L_AES_CTR_encrypt_start_block_192 -L_AES_CTR_encrypt_loop_block_256: + bne L_AES_CTR_encrypt_loop_block_256 + b L_AES_CTR_encrypt_end +L_AES_CTR_encrypt_start_block_192: +L_AES_CTR_encrypt_loop_block_192: push {r1, r2, lr} ldr lr, [sp, #16] adds r11, r7, #1 @@ -14084,11 +6677,11 @@ L_AES_CTR_encrypt_loop_block_256: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #6 + mov r1, #5 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_CTR_encrypt_block_nr_256: +L_AES_CTR_encrypt_block_nr_192: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -14406,7 +6999,7 @@ L_AES_CTR_encrypt_block_nr_256: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_CTR_encrypt_block_nr_256 + bne L_AES_CTR_encrypt_block_nr_192 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -14766,10 +7359,10 @@ L_AES_CTR_encrypt_block_nr_256: subs r2, r2, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_CTR_encrypt_loop_block_256 + bne L_AES_CTR_encrypt_loop_block_192 b L_AES_CTR_encrypt_end -L_AES_CTR_encrypt_start_block_192: -L_AES_CTR_encrypt_loop_block_192: +L_AES_CTR_encrypt_start_block_128: +L_AES_CTR_encrypt_loop_block_128: push {r1, r2, lr} ldr lr, [sp, #16] adds r11, r7, #1 @@ -14783,11 +7376,11 @@ L_AES_CTR_encrypt_loop_block_192: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #5 + mov r1, #4 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_CTR_encrypt_block_nr_192: +L_AES_CTR_encrypt_block_nr_128: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -15105,7 +7698,7 @@ L_AES_CTR_encrypt_block_nr_192: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_CTR_encrypt_block_nr_192 + bne L_AES_CTR_encrypt_block_nr_128 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -15456,190 +8049,209 @@ L_AES_CTR_encrypt_block_nr_192: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - ldr r8, [sp, #4] - str r4, [r1] - str r5, [r1, #4] - str r6, [r1, #8] - str r7, [r1, #12] - ldm r8, {r4, r5, r6, r7} - subs r2, r2, #16 - add lr, lr, #16 - add r1, r1, #16 - bne L_AES_CTR_encrypt_loop_block_192 - b L_AES_CTR_encrypt_end -L_AES_CTR_encrypt_start_block_128: -L_AES_CTR_encrypt_loop_block_128: - push {r1, r2, lr} - ldr lr, [sp, #16] - adds r11, r7, #1 - adcs r10, r6, #0 - adcs r9, r5, #0 - adc r8, r4, #0 - stm lr, {r8, r9, r10, r11} - ldm r3!, {r8, r9, r10, r11} - # Round: 0 - XOR in key schedule - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 - mov r1, #4 -#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE - bl AES_encrypt_block + ldr r8, [sp, #4] + str r4, [r1] + str r5, [r1, #4] + str r6, [r1, #8] + str r7, [r1, #12] + ldm r8, {r4, r5, r6, r7} + subs r2, r2, #16 + add lr, lr, #16 + add r1, r1, #16 + bne L_AES_CTR_encrypt_loop_block_128 +L_AES_CTR_encrypt_end: + pop {r3, r8} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + eor r10, r4, r4, ror #16 + eor r11, r5, r5, ror #16 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 + eor r4, r4, r10, lsr #8 + eor r5, r5, r11, lsr #8 + eor r10, r6, r6, ror #16 + eor r11, r7, r7, ror #16 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r6, r6, #8 + ror r7, r7, #8 + eor r6, r6, r10, lsr #8 + eor r7, r7, r11, lsr #8 #else -L_AES_CTR_encrypt_block_nr_128: + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r8, {r4, r5, r6, r7} + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size AES_CTR_encrypt,.-AES_CTR_encrypt +#endif /* WOLFSSL_AES_COUNTER */ +#ifdef HAVE_AES_DECRYPT +#if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || \ + defined(HAVE_AES_CBC) || defined(HAVE_AES_ECB) +#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE + .text + .align 4 + .globl AES_decrypt_block + .type AES_decrypt_block, %function +AES_decrypt_block: + push {lr} +L_AES_decrypt_block_nr: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r5, #8 + lsl r8, r7, #8 lsr r8, r8, #24 #else - uxtb r8, r5, ror #16 + uxtb r8, r7, ror #16 #endif #else - ubfx r8, r5, #16, #8 + ubfx r8, r7, #16, #8 #endif lsr r11, r4, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r6, #16 - lsr lr, lr, #24 + lsl r12, r6, #16 + lsr r12, r12, #24 #else - uxtb lr, r6, ror #8 + uxtb r12, r6, ror #8 #endif #else - ubfx lr, r6, #8, #8 + ubfx r12, r6, #8, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r7, #24 - lsr r2, r2, #24 + lsl lr, r5, #24 + lsr lr, lr, #24 #else - uxtb r2, r7 + uxtb lr, r5 #endif #else - ubfx r2, r7, #0, #8 + ubfx lr, r5, #0, #8 #endif ldr r8, [r0, r8, lsl #2] ldr r11, [r0, r11, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r9, r6, #8 + lsl r9, r4, #8 lsr r9, r9, #24 #else - uxtb r9, r6, ror #16 + uxtb r9, r4, ror #16 #endif #else - ubfx r9, r6, #16, #8 + ubfx r9, r4, #16, #8 #endif eor r8, r8, r11, ror #24 lsr r11, r5, #24 - eor r8, r8, lr, ror #8 + eor r8, r8, r12, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r7, #16 - lsr lr, lr, #24 + lsl r12, r7, #16 + lsr r12, r12, #24 #else - uxtb lr, r7, ror #8 + uxtb r12, r7, ror #8 #endif #else - ubfx lr, r7, #8, #8 + ubfx r12, r7, #8, #8 #endif - eor r8, r8, r2, ror #16 + eor r8, r8, lr, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r4, #24 - lsr r2, r2, #24 + lsl lr, r6, #24 + lsr lr, lr, #24 #else - uxtb r2, r4 + uxtb lr, r6 #endif #else - ubfx r2, r4, #0, #8 + ubfx lr, r6, #0, #8 #endif ldr r9, [r0, r9, lsl #2] ldr r11, [r0, r11, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r10, r7, #8 + lsl r10, r5, #8 lsr r10, r10, #24 #else - uxtb r10, r7, ror #16 + uxtb r10, r5, ror #16 #endif #else - ubfx r10, r7, #16, #8 + ubfx r10, r5, #16, #8 #endif eor r9, r9, r11, ror #24 lsr r11, r6, #24 - eor r9, r9, lr, ror #8 + eor r9, r9, r12, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r4, #16 - lsr lr, lr, #24 + lsl r12, r4, #16 + lsr r12, r12, #24 #else - uxtb lr, r4, ror #8 + uxtb r12, r4, ror #8 #endif #else - ubfx lr, r4, #8, #8 + ubfx r12, r4, #8, #8 #endif - eor r9, r9, r2, ror #16 + eor r9, r9, lr, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r5, #24 - lsr r2, r2, #24 + lsl lr, r7, #24 + lsr lr, lr, #24 #else - uxtb r2, r5 + uxtb lr, r7 #endif #else - ubfx r2, r5, #0, #8 + ubfx lr, r7, #0, #8 #endif ldr r10, [r0, r10, lsl #2] ldr r11, [r0, r11, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r6, #24 - lsr r6, r6, #24 + lsl r4, r4, #24 + lsr r4, r4, #24 #else - uxtb r6, r6 + uxtb r4, r4 #endif #else - ubfx r6, r6, #0, #8 + ubfx r4, r4, #0, #8 #endif eor r10, r10, r11, ror #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r11, r4, #8 + lsl r11, r6, #8 lsr r11, r11, #24 #else - uxtb r11, r4, ror #16 + uxtb r11, r6, ror #16 #endif #else - ubfx r11, r4, #16, #8 + ubfx r11, r6, #16, #8 #endif - eor r10, r10, lr, ror #8 - lsr lr, r7, #24 - eor r10, r10, r2, ror #16 + eor r10, r10, r12, ror #8 + lsr r12, r7, #24 + eor r10, r10, lr, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r5, #16 - lsr r2, r2, #24 + lsl lr, r5, #16 + lsr lr, lr, #24 #else - uxtb r2, r5, ror #8 + uxtb lr, r5, ror #8 #endif #else - ubfx r2, r5, #8, #8 + ubfx lr, r5, #8, #8 #endif - ldr r6, [r0, r6, lsl #2] - ldr lr, [r0, lr, lsl #2] + ldr r4, [r0, r4, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr r11, [r0, r11, lsl #2] - ldr r2, [r0, r2, lsl #2] - eor lr, lr, r6, ror #24 + ldr lr, [r0, lr, lsl #2] + eor r12, r12, r4, ror #24 ldm r3!, {r4, r5, r6, r7} - eor r11, r11, lr, ror #24 - eor r11, r11, r2, ror #8 + eor r11, r11, lr, ror #8 + eor r11, r11, r12, ror #24 # XOR in Key Schedule eor r8, r8, r4 eor r9, r9, r5 @@ -15647,317 +8259,317 @@ L_AES_CTR_encrypt_block_nr_128: eor r11, r11, r7 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r9, #8 + lsl r4, r11, #8 lsr r4, r4, #24 #else - uxtb r4, r9, ror #16 + uxtb r4, r11, ror #16 #endif #else - ubfx r4, r9, #16, #8 + ubfx r4, r11, #16, #8 #endif lsr r7, r8, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r10, #16 - lsr lr, lr, #24 + lsl r12, r10, #16 + lsr r12, r12, #24 #else - uxtb lr, r10, ror #8 + uxtb r12, r10, ror #8 #endif #else - ubfx lr, r10, #8, #8 + ubfx r12, r10, #8, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r11, #24 - lsr r2, r2, #24 + lsl lr, r9, #24 + lsr lr, lr, #24 #else - uxtb r2, r11 + uxtb lr, r9 #endif #else - ubfx r2, r11, #0, #8 + ubfx lr, r9, #0, #8 #endif ldr r4, [r0, r4, lsl #2] ldr r7, [r0, r7, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r10, #8 + lsl r5, r8, #8 lsr r5, r5, #24 #else - uxtb r5, r10, ror #16 + uxtb r5, r8, ror #16 #endif #else - ubfx r5, r10, #16, #8 + ubfx r5, r8, #16, #8 #endif eor r4, r4, r7, ror #24 lsr r7, r9, #24 - eor r4, r4, lr, ror #8 + eor r4, r4, r12, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r11, #16 - lsr lr, lr, #24 + lsl r12, r11, #16 + lsr r12, r12, #24 #else - uxtb lr, r11, ror #8 + uxtb r12, r11, ror #8 #endif #else - ubfx lr, r11, #8, #8 + ubfx r12, r11, #8, #8 #endif - eor r4, r4, r2, ror #16 + eor r4, r4, lr, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r8, #24 - lsr r2, r2, #24 + lsl lr, r10, #24 + lsr lr, lr, #24 #else - uxtb r2, r8 + uxtb lr, r10 #endif #else - ubfx r2, r8, #0, #8 + ubfx lr, r10, #0, #8 #endif ldr r5, [r0, r5, lsl #2] ldr r7, [r0, r7, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r11, #8 + lsl r6, r9, #8 lsr r6, r6, #24 #else - uxtb r6, r11, ror #16 + uxtb r6, r9, ror #16 #endif #else - ubfx r6, r11, #16, #8 + ubfx r6, r9, #16, #8 #endif eor r5, r5, r7, ror #24 lsr r7, r10, #24 - eor r5, r5, lr, ror #8 + eor r5, r5, r12, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r8, #16 - lsr lr, lr, #24 + lsl r12, r8, #16 + lsr r12, r12, #24 #else - uxtb lr, r8, ror #8 + uxtb r12, r8, ror #8 #endif #else - ubfx lr, r8, #8, #8 + ubfx r12, r8, #8, #8 #endif - eor r5, r5, r2, ror #16 + eor r5, r5, lr, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r9, #24 - lsr r2, r2, #24 + lsl lr, r11, #24 + lsr lr, lr, #24 #else - uxtb r2, r9 + uxtb lr, r11 #endif #else - ubfx r2, r9, #0, #8 + ubfx lr, r11, #0, #8 #endif ldr r6, [r0, r6, lsl #2] ldr r7, [r0, r7, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r10, r10, #24 - lsr r10, r10, #24 + lsl r8, r8, #24 + lsr r8, r8, #24 #else - uxtb r10, r10 + uxtb r8, r8 #endif #else - ubfx r10, r10, #0, #8 + ubfx r8, r8, #0, #8 #endif eor r6, r6, r7, ror #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r8, #8 + lsl r7, r10, #8 lsr r7, r7, #24 #else - uxtb r7, r8, ror #16 + uxtb r7, r10, ror #16 #endif #else - ubfx r7, r8, #16, #8 + ubfx r7, r10, #16, #8 #endif - eor r6, r6, lr, ror #8 - lsr lr, r11, #24 - eor r6, r6, r2, ror #16 + eor r6, r6, r12, ror #8 + lsr r12, r11, #24 + eor r6, r6, lr, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r9, #16 - lsr r2, r2, #24 + lsl lr, r9, #16 + lsr lr, lr, #24 #else - uxtb r2, r9, ror #8 + uxtb lr, r9, ror #8 #endif #else - ubfx r2, r9, #8, #8 + ubfx lr, r9, #8, #8 #endif - ldr r10, [r0, r10, lsl #2] - ldr lr, [r0, lr, lsl #2] + ldr r8, [r0, r8, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr r7, [r0, r7, lsl #2] - ldr r2, [r0, r2, lsl #2] - eor lr, lr, r10, ror #24 + ldr lr, [r0, lr, lsl #2] + eor r12, r12, r8, ror #24 ldm r3!, {r8, r9, r10, r11} - eor r7, r7, lr, ror #24 - eor r7, r7, r2, ror #8 + eor r7, r7, lr, ror #8 + eor r7, r7, r12, ror #24 # XOR in Key Schedule eor r4, r4, r8 eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_CTR_encrypt_block_nr_128 + bne L_AES_decrypt_block_nr #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r5, #8 + lsl r8, r7, #8 lsr r8, r8, #24 #else - uxtb r8, r5, ror #16 + uxtb r8, r7, ror #16 #endif #else - ubfx r8, r5, #16, #8 + ubfx r8, r7, #16, #8 #endif lsr r11, r4, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r6, #16 - lsr lr, lr, #24 + lsl r12, r6, #16 + lsr r12, r12, #24 #else - uxtb lr, r6, ror #8 + uxtb r12, r6, ror #8 #endif #else - ubfx lr, r6, #8, #8 + ubfx r12, r6, #8, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r7, #24 - lsr r2, r2, #24 + lsl lr, r5, #24 + lsr lr, lr, #24 #else - uxtb r2, r7 + uxtb lr, r5 #endif #else - ubfx r2, r7, #0, #8 + ubfx lr, r5, #0, #8 #endif ldr r8, [r0, r8, lsl #2] ldr r11, [r0, r11, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r9, r6, #8 + lsl r9, r4, #8 lsr r9, r9, #24 #else - uxtb r9, r6, ror #16 + uxtb r9, r4, ror #16 #endif #else - ubfx r9, r6, #16, #8 + ubfx r9, r4, #16, #8 #endif eor r8, r8, r11, ror #24 lsr r11, r5, #24 - eor r8, r8, lr, ror #8 + eor r8, r8, r12, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r7, #16 - lsr lr, lr, #24 + lsl r12, r7, #16 + lsr r12, r12, #24 #else - uxtb lr, r7, ror #8 + uxtb r12, r7, ror #8 #endif #else - ubfx lr, r7, #8, #8 + ubfx r12, r7, #8, #8 #endif - eor r8, r8, r2, ror #16 + eor r8, r8, lr, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r4, #24 - lsr r2, r2, #24 + lsl lr, r6, #24 + lsr lr, lr, #24 #else - uxtb r2, r4 + uxtb lr, r6 #endif #else - ubfx r2, r4, #0, #8 + ubfx lr, r6, #0, #8 #endif ldr r9, [r0, r9, lsl #2] ldr r11, [r0, r11, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r10, r7, #8 + lsl r10, r5, #8 lsr r10, r10, #24 #else - uxtb r10, r7, ror #16 + uxtb r10, r5, ror #16 #endif #else - ubfx r10, r7, #16, #8 + ubfx r10, r5, #16, #8 #endif eor r9, r9, r11, ror #24 lsr r11, r6, #24 - eor r9, r9, lr, ror #8 + eor r9, r9, r12, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r4, #16 - lsr lr, lr, #24 + lsl r12, r4, #16 + lsr r12, r12, #24 #else - uxtb lr, r4, ror #8 + uxtb r12, r4, ror #8 #endif #else - ubfx lr, r4, #8, #8 + ubfx r12, r4, #8, #8 #endif - eor r9, r9, r2, ror #16 + eor r9, r9, lr, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r5, #24 - lsr r2, r2, #24 + lsl lr, r7, #24 + lsr lr, lr, #24 #else - uxtb r2, r5 + uxtb lr, r7 #endif #else - ubfx r2, r5, #0, #8 + ubfx lr, r7, #0, #8 #endif ldr r10, [r0, r10, lsl #2] ldr r11, [r0, r11, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r6, #24 - lsr r6, r6, #24 + lsl r4, r4, #24 + lsr r4, r4, #24 #else - uxtb r6, r6 + uxtb r4, r4 #endif #else - ubfx r6, r6, #0, #8 + ubfx r4, r4, #0, #8 #endif eor r10, r10, r11, ror #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r11, r4, #8 + lsl r11, r6, #8 lsr r11, r11, #24 #else - uxtb r11, r4, ror #16 + uxtb r11, r6, ror #16 #endif #else - ubfx r11, r4, #16, #8 + ubfx r11, r6, #16, #8 #endif - eor r10, r10, lr, ror #8 - lsr lr, r7, #24 - eor r10, r10, r2, ror #16 + eor r10, r10, r12, ror #8 + lsr r12, r7, #24 + eor r10, r10, lr, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r5, #16 - lsr r2, r2, #24 + lsl lr, r5, #16 + lsr lr, lr, #24 #else - uxtb r2, r5, ror #8 + uxtb lr, r5, ror #8 #endif #else - ubfx r2, r5, #8, #8 + ubfx lr, r5, #8, #8 #endif - ldr r6, [r0, r6, lsl #2] - ldr lr, [r0, lr, lsl #2] + ldr r4, [r0, r4, lsl #2] + ldr r12, [r0, r12, lsl #2] ldr r11, [r0, r11, lsl #2] - ldr r2, [r0, r2, lsl #2] - eor lr, lr, r6, ror #24 + ldr lr, [r0, lr, lsl #2] + eor r12, r12, r4, ror #24 ldm r3!, {r4, r5, r6, r7} - eor r11, r11, lr, ror #24 - eor r11, r11, r2, ror #8 + eor r11, r11, lr, ror #8 + eor r11, r11, r12, ror #24 # XOR in Key Schedule eor r8, r8, r4 eor r9, r9, r5 @@ -15965,13 +8577,13 @@ L_AES_CTR_encrypt_block_nr_128: eor r11, r11, r7 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r11, #24 + lsl r4, r9, #24 lsr r4, r4, #24 #else - uxtb r4, r11 + uxtb r4, r9 #endif #else - ubfx r4, r11, #0, #8 + ubfx r4, r9, #0, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) @@ -15985,28 +8597,28 @@ L_AES_CTR_encrypt_block_nr_128: #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r9, #8 - lsr lr, lr, #24 + lsl r12, r11, #8 + lsr r12, r12, #24 #else - uxtb lr, r9, ror #16 + uxtb r12, r11, ror #16 #endif #else - ubfx lr, r9, #16, #8 + ubfx r12, r11, #16, #8 #endif - lsr r2, r8, #24 - ldrb r4, [r0, r4, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] + lsr lr, r8, #24 + ldrb r4, [r2, r4] + ldrb r7, [r2, r7] + ldrb r12, [r2, r12] + ldrb lr, [r2, lr] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r8, #24 + lsl r5, r10, #24 lsr r5, r5, #24 #else - uxtb r5, r8 + uxtb r5, r10 #endif #else - ubfx r5, r8, #0, #8 + ubfx r5, r10, #0, #8 #endif eor r4, r4, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) @@ -16019,32 +8631,32 @@ L_AES_CTR_encrypt_block_nr_128: #else ubfx r7, r11, #8, #8 #endif - eor r4, r4, lr, lsl #16 + eor r4, r4, r12, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r10, #8 - lsr lr, lr, #24 + lsl r12, r8, #8 + lsr r12, r12, #24 #else - uxtb lr, r10, ror #16 + uxtb r12, r8, ror #16 #endif #else - ubfx lr, r10, #16, #8 + ubfx r12, r8, #16, #8 #endif - eor r4, r4, r2, lsl #24 - lsr r2, r9, #24 - ldrb r5, [r0, r5, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] + eor r4, r4, lr, lsl #24 + lsr lr, r9, #24 + ldrb r7, [r2, r7] + ldrb lr, [r2, lr] + ldrb r5, [r2, r5] + ldrb r12, [r2, r12] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r9, #24 + lsl r6, r11, #24 lsr r6, r6, #24 #else - uxtb r6, r9 + uxtb r6, r11 #endif #else - ubfx r6, r9, #0, #8 + ubfx r6, r11, #0, #8 #endif eor r5, r5, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) @@ -16057,73 +8669,156 @@ L_AES_CTR_encrypt_block_nr_128: #else ubfx r7, r8, #8, #8 #endif - eor r5, r5, lr, lsl #16 + eor r5, r5, r12, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r11, #8 - lsr lr, lr, #24 + lsl r12, r9, #8 + lsr r12, r12, #24 #else - uxtb lr, r11, ror #16 + uxtb r12, r9, ror #16 #endif #else - ubfx lr, r11, #16, #8 + ubfx r12, r9, #16, #8 #endif - eor r5, r5, r2, lsl #24 - lsr r2, r10, #24 - ldrb r6, [r0, r6, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] + eor r5, r5, lr, lsl #24 + lsr lr, r10, #24 + ldrb r7, [r2, r7] + ldrb lr, [r2, lr] + ldrb r6, [r2, r6] + ldrb r12, [r2, r12] lsr r11, r11, #24 eor r6, r6, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r10, #24 + lsl r7, r8, #24 lsr r7, r7, #24 #else - uxtb r7, r10 + uxtb r7, r8 #endif #else - ubfx r7, r10, #0, #8 + ubfx r7, r8, #0, #8 #endif - eor r6, r6, lr, lsl #16 + eor r6, r6, r12, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r9, #16 - lsr lr, lr, #24 + lsl r12, r9, #16 + lsr r12, r12, #24 #else - uxtb lr, r9, ror #8 + uxtb r12, r9, ror #8 #endif #else - ubfx lr, r9, #8, #8 + ubfx r12, r9, #8, #8 #endif - eor r6, r6, r2, lsl #24 + eor r6, r6, lr, lsl #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r8, #8 - lsr r2, r2, #24 + lsl lr, r10, #8 + lsr lr, lr, #24 #else - uxtb r2, r8, ror #16 + uxtb lr, r10, ror #16 #endif #else - ubfx r2, r8, #16, #8 + ubfx lr, r10, #16, #8 #endif - ldrb r11, [r0, r11, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] - eor lr, lr, r11, lsl #16 + ldrb r11, [r2, r11] + ldrb r12, [r2, r12] + ldrb r7, [r2, r7] + ldrb lr, [r2, lr] + eor r12, r12, r11, lsl #16 ldm r3, {r8, r9, r10, r11} - eor r7, r7, lr, lsl #8 - eor r7, r7, r2, lsl #16 + eor r7, r7, r12, lsl #8 + eor r7, r7, lr, lsl #16 # XOR in Key Schedule eor r4, r4, r8 eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 + pop {pc} + .size AES_decrypt_block,.-AES_decrypt_block #endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - pop {r1, r2, lr} - ldr r3, [sp] +#ifndef __APPLE__ + .text + .type L_AES_ARM32_td_ecb, %object + .size L_AES_ARM32_td_ecb, 12 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_AES_ARM32_td_ecb: + .long L_AES_ARM32_td_data +#if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AES_ECB) +#ifndef __APPLE__ + .text + .type L_AES_ARM32_ecb_td4, %object + .size L_AES_ARM32_ecb_td4, 256 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_AES_ARM32_ecb_td4: + .byte 0x52,0x09,0x6a,0xd5,0x30,0x36,0xa5,0x38 + .byte 0xbf,0x40,0xa3,0x9e,0x81,0xf3,0xd7,0xfb + .byte 0x7c,0xe3,0x39,0x82,0x9b,0x2f,0xff,0x87 + .byte 0x34,0x8e,0x43,0x44,0xc4,0xde,0xe9,0xcb + .byte 0x54,0x7b,0x94,0x32,0xa6,0xc2,0x23,0x3d + .byte 0xee,0x4c,0x95,0x0b,0x42,0xfa,0xc3,0x4e + .byte 0x08,0x2e,0xa1,0x66,0x28,0xd9,0x24,0xb2 + .byte 0x76,0x5b,0xa2,0x49,0x6d,0x8b,0xd1,0x25 + .byte 0x72,0xf8,0xf6,0x64,0x86,0x68,0x98,0x16 + .byte 0xd4,0xa4,0x5c,0xcc,0x5d,0x65,0xb6,0x92 + .byte 0x6c,0x70,0x48,0x50,0xfd,0xed,0xb9,0xda + .byte 0x5e,0x15,0x46,0x57,0xa7,0x8d,0x9d,0x84 + .byte 0x90,0xd8,0xab,0x00,0x8c,0xbc,0xd3,0x0a + .byte 0xf7,0xe4,0x58,0x05,0xb8,0xb3,0x45,0x06 + .byte 0xd0,0x2c,0x1e,0x8f,0xca,0x3f,0x0f,0x02 + .byte 0xc1,0xaf,0xbd,0x03,0x01,0x13,0x8a,0x6b + .byte 0x3a,0x91,0x11,0x41,0x4f,0x67,0xdc,0xea + .byte 0x97,0xf2,0xcf,0xce,0xf0,0xb4,0xe6,0x73 + .byte 0x96,0xac,0x74,0x22,0xe7,0xad,0x35,0x85 + .byte 0xe2,0xf9,0x37,0xe8,0x1c,0x75,0xdf,0x6e + .byte 0x47,0xf1,0x1a,0x71,0x1d,0x29,0xc5,0x89 + .byte 0x6f,0xb7,0x62,0x0e,0xaa,0x18,0xbe,0x1b + .byte 0xfc,0x56,0x3e,0x4b,0xc6,0xd2,0x79,0x20 + .byte 0x9a,0xdb,0xc0,0xfe,0x78,0xcd,0x5a,0xf4 + .byte 0x1f,0xdd,0xa8,0x33,0x88,0x07,0xc7,0x31 + .byte 0xb1,0x12,0x10,0x59,0x27,0x80,0xec,0x5f + .byte 0x60,0x51,0x7f,0xa9,0x19,0xb5,0x4a,0x0d + .byte 0x2d,0xe5,0x7a,0x9f,0x93,0xc9,0x9c,0xef + .byte 0xa0,0xe0,0x3b,0x4d,0xae,0x2a,0xf5,0xb0 + .byte 0xc8,0xeb,0xbb,0x3c,0x83,0x53,0x99,0x61 + .byte 0x17,0x2b,0x04,0x7e,0xba,0x77,0xd6,0x26 + .byte 0xe1,0x69,0x14,0x63,0x55,0x21,0x0c,0x7d + .text + .align 4 + .globl AES_ECB_decrypt + .type AES_ECB_decrypt, %function +AES_ECB_decrypt: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + ldr r8, [sp, #36] + mov lr, r0 + adr r0, L_AES_ARM32_td_ecb + ldr r0, [r0] + mov r12, r2 + adr r2, L_AES_ARM32_ecb_td4 + cmp r8, #10 + beq L_AES_ECB_decrypt_start_block_128 + cmp r8, #12 + beq L_AES_ECB_decrypt_start_block_192 +L_AES_ECB_decrypt_loop_block_256: + ldr r4, [lr] + ldr r5, [lr, #4] + ldr r6, [lr, #8] + ldr r7, [lr, #12] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) eor r8, r4, r4, ror #16 eor r9, r5, r5, ror #16 @@ -16147,64 +8842,18 @@ L_AES_CTR_encrypt_block_nr_128: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - ldr r8, [lr] - ldr r9, [lr, #4] - ldr r10, [lr, #8] - ldr r11, [lr, #12] + push {r1, r3, r12, lr} + ldm r3!, {r8, r9, r10, r11} + # Round: 0 - XOR in key schedule eor r4, r4, r8 eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - ldr r8, [sp, #4] - str r4, [r1] - str r5, [r1, #4] - str r6, [r1, #8] - str r7, [r1, #12] - ldm r8, {r4, r5, r6, r7} - subs r2, r2, #16 - add lr, lr, #16 - add r1, r1, #16 - bne L_AES_CTR_encrypt_loop_block_128 -L_AES_CTR_encrypt_end: - pop {r3, r8} -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r10, r4, r4, ror #16 - eor r11, r5, r5, ror #16 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - eor r4, r4, r10, lsr #8 - eor r5, r5, r11, lsr #8 - eor r10, r6, r6, ror #16 - eor r11, r7, r7, ror #16 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r6, r6, #8 - ror r7, r7, #8 - eor r6, r6, r10, lsr #8 - eor r7, r7, r11, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - stm r8, {r4, r5, r6, r7} - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size AES_CTR_encrypt,.-AES_CTR_encrypt -#endif /* WOLFSSL_AES_COUNTER */ -#ifdef HAVE_AES_DECRYPT -#if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || \ - defined(HAVE_AES_CBC) || defined(HAVE_AES_ECB) + mov r1, #6 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE - .text - .align 4 - .globl AES_decrypt_block - .type AES_decrypt_block, %function -AES_decrypt_block: - push {lr} -L_AES_decrypt_block_nr: + bl AES_decrypt_block +#else +L_AES_ECB_decrypt_block_nr_256: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -16522,7 +9171,7 @@ L_AES_decrypt_block_nr: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_decrypt_block_nr + bne L_AES_ECB_decrypt_block_nr_256 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -16839,88 +9488,42 @@ L_AES_decrypt_block_nr: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - pop {pc} - .size AES_decrypt_block,.-AES_decrypt_block #endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ -#ifndef __APPLE__ - .text - .type L_AES_ARM32_td_ecb, %object - .size L_AES_ARM32_td_ecb, 12 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 -#else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_ARM32_td_ecb: - .long L_AES_ARM32_td_data -#if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AES_ECB) -#ifndef __APPLE__ - .text - .type L_AES_ARM32_ecb_td4, %object - .size L_AES_ARM32_ecb_td4, 256 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 4-byte aligned, 32-bit aligned -#ifndef __APPLE__ - .align 2 -#else - .p2align 2 -#endif /* __APPLE__ */ -L_AES_ARM32_ecb_td4: - .byte 0x52,0x09,0x6a,0xd5,0x30,0x36,0xa5,0x38 - .byte 0xbf,0x40,0xa3,0x9e,0x81,0xf3,0xd7,0xfb - .byte 0x7c,0xe3,0x39,0x82,0x9b,0x2f,0xff,0x87 - .byte 0x34,0x8e,0x43,0x44,0xc4,0xde,0xe9,0xcb - .byte 0x54,0x7b,0x94,0x32,0xa6,0xc2,0x23,0x3d - .byte 0xee,0x4c,0x95,0x0b,0x42,0xfa,0xc3,0x4e - .byte 0x08,0x2e,0xa1,0x66,0x28,0xd9,0x24,0xb2 - .byte 0x76,0x5b,0xa2,0x49,0x6d,0x8b,0xd1,0x25 - .byte 0x72,0xf8,0xf6,0x64,0x86,0x68,0x98,0x16 - .byte 0xd4,0xa4,0x5c,0xcc,0x5d,0x65,0xb6,0x92 - .byte 0x6c,0x70,0x48,0x50,0xfd,0xed,0xb9,0xda - .byte 0x5e,0x15,0x46,0x57,0xa7,0x8d,0x9d,0x84 - .byte 0x90,0xd8,0xab,0x00,0x8c,0xbc,0xd3,0x0a - .byte 0xf7,0xe4,0x58,0x05,0xb8,0xb3,0x45,0x06 - .byte 0xd0,0x2c,0x1e,0x8f,0xca,0x3f,0x0f,0x02 - .byte 0xc1,0xaf,0xbd,0x03,0x01,0x13,0x8a,0x6b - .byte 0x3a,0x91,0x11,0x41,0x4f,0x67,0xdc,0xea - .byte 0x97,0xf2,0xcf,0xce,0xf0,0xb4,0xe6,0x73 - .byte 0x96,0xac,0x74,0x22,0xe7,0xad,0x35,0x85 - .byte 0xe2,0xf9,0x37,0xe8,0x1c,0x75,0xdf,0x6e - .byte 0x47,0xf1,0x1a,0x71,0x1d,0x29,0xc5,0x89 - .byte 0x6f,0xb7,0x62,0x0e,0xaa,0x18,0xbe,0x1b - .byte 0xfc,0x56,0x3e,0x4b,0xc6,0xd2,0x79,0x20 - .byte 0x9a,0xdb,0xc0,0xfe,0x78,0xcd,0x5a,0xf4 - .byte 0x1f,0xdd,0xa8,0x33,0x88,0x07,0xc7,0x31 - .byte 0xb1,0x12,0x10,0x59,0x27,0x80,0xec,0x5f - .byte 0x60,0x51,0x7f,0xa9,0x19,0xb5,0x4a,0x0d - .byte 0x2d,0xe5,0x7a,0x9f,0x93,0xc9,0x9c,0xef - .byte 0xa0,0xe0,0x3b,0x4d,0xae,0x2a,0xf5,0xb0 - .byte 0xc8,0xeb,0xbb,0x3c,0x83,0x53,0x99,0x61 - .byte 0x17,0x2b,0x04,0x7e,0xba,0x77,0xd6,0x26 - .byte 0xe1,0x69,0x14,0x63,0x55,0x21,0x0c,0x7d - .text - .align 4 - .globl AES_ECB_decrypt - .type AES_ECB_decrypt, %function -AES_ECB_decrypt: - push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - ldr r8, [sp, #36] - mov lr, r0 - adr r0, L_AES_ARM32_td_ecb - ldr r0, [r0] - mov r12, r2 - adr r2, L_AES_ARM32_ecb_td4 - cmp r8, #10 - beq L_AES_ECB_decrypt_start_block_128 - cmp r8, #12 - beq L_AES_ECB_decrypt_start_block_192 -L_AES_ECB_decrypt_loop_block_256: + pop {r1, r3, r12, lr} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + eor r8, r4, r4, ror #16 + eor r9, r5, r5, ror #16 + eor r10, r6, r6, ror #16 + eor r11, r7, r7, ror #16 + bic r8, r8, #0xff0000 + bic r9, r9, #0xff0000 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 + ror r6, r6, #8 + ror r7, r7, #8 + eor r4, r4, r8, lsr #8 + eor r5, r5, r9, lsr #8 + eor r6, r6, r10, lsr #8 + eor r7, r7, r11, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + str r4, [r1] + str r5, [r1, #4] + str r6, [r1, #8] + str r7, [r1, #12] + subs r12, r12, #16 + add lr, lr, #16 + add r1, r1, #16 + bne L_AES_ECB_decrypt_loop_block_256 + b L_AES_ECB_decrypt_end +L_AES_ECB_decrypt_start_block_192: +L_AES_ECB_decrypt_loop_block_192: ldr r4, [lr] ldr r5, [lr, #4] ldr r6, [lr, #8] @@ -16955,11 +9558,11 @@ L_AES_ECB_decrypt_loop_block_256: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #6 + mov r1, #5 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_decrypt_block #else -L_AES_ECB_decrypt_block_nr_256: +L_AES_ECB_decrypt_block_nr_192: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -17277,7 +9880,7 @@ L_AES_ECB_decrypt_block_nr_256: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_ECB_decrypt_block_nr_256 + bne L_AES_ECB_decrypt_block_nr_192 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -17626,10 +10229,10 @@ L_AES_ECB_decrypt_block_nr_256: subs r12, r12, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_ECB_decrypt_loop_block_256 + bne L_AES_ECB_decrypt_loop_block_192 b L_AES_ECB_decrypt_end -L_AES_ECB_decrypt_start_block_192: -L_AES_ECB_decrypt_loop_block_192: +L_AES_ECB_decrypt_start_block_128: +L_AES_ECB_decrypt_loop_block_128: ldr r4, [lr] ldr r5, [lr, #4] ldr r6, [lr, #8] @@ -17664,11 +10267,11 @@ L_AES_ECB_decrypt_loop_block_192: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #5 + mov r1, #4 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_decrypt_block #else -L_AES_ECB_decrypt_block_nr_192: +L_AES_ECB_decrypt_block_nr_128: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -17986,7 +10589,7 @@ L_AES_ECB_decrypt_block_nr_192: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_ECB_decrypt_block_nr_192 + bne L_AES_ECB_decrypt_block_nr_128 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -18335,14 +10938,95 @@ L_AES_ECB_decrypt_block_nr_192: subs r12, r12, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_ECB_decrypt_loop_block_192 - b L_AES_ECB_decrypt_end -L_AES_ECB_decrypt_start_block_128: -L_AES_ECB_decrypt_loop_block_128: + bne L_AES_ECB_decrypt_loop_block_128 +L_AES_ECB_decrypt_end: + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size AES_ECB_decrypt,.-AES_ECB_decrypt +#endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER || defined(HAVE_AES_ECB) */ +#ifdef HAVE_AES_CBC +#ifndef __APPLE__ + .text + .type L_AES_ARM32_cbc_td4, %object + .size L_AES_ARM32_cbc_td4, 256 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_AES_ARM32_cbc_td4: + .byte 0x52,0x09,0x6a,0xd5,0x30,0x36,0xa5,0x38 + .byte 0xbf,0x40,0xa3,0x9e,0x81,0xf3,0xd7,0xfb + .byte 0x7c,0xe3,0x39,0x82,0x9b,0x2f,0xff,0x87 + .byte 0x34,0x8e,0x43,0x44,0xc4,0xde,0xe9,0xcb + .byte 0x54,0x7b,0x94,0x32,0xa6,0xc2,0x23,0x3d + .byte 0xee,0x4c,0x95,0x0b,0x42,0xfa,0xc3,0x4e + .byte 0x08,0x2e,0xa1,0x66,0x28,0xd9,0x24,0xb2 + .byte 0x76,0x5b,0xa2,0x49,0x6d,0x8b,0xd1,0x25 + .byte 0x72,0xf8,0xf6,0x64,0x86,0x68,0x98,0x16 + .byte 0xd4,0xa4,0x5c,0xcc,0x5d,0x65,0xb6,0x92 + .byte 0x6c,0x70,0x48,0x50,0xfd,0xed,0xb9,0xda + .byte 0x5e,0x15,0x46,0x57,0xa7,0x8d,0x9d,0x84 + .byte 0x90,0xd8,0xab,0x00,0x8c,0xbc,0xd3,0x0a + .byte 0xf7,0xe4,0x58,0x05,0xb8,0xb3,0x45,0x06 + .byte 0xd0,0x2c,0x1e,0x8f,0xca,0x3f,0x0f,0x02 + .byte 0xc1,0xaf,0xbd,0x03,0x01,0x13,0x8a,0x6b + .byte 0x3a,0x91,0x11,0x41,0x4f,0x67,0xdc,0xea + .byte 0x97,0xf2,0xcf,0xce,0xf0,0xb4,0xe6,0x73 + .byte 0x96,0xac,0x74,0x22,0xe7,0xad,0x35,0x85 + .byte 0xe2,0xf9,0x37,0xe8,0x1c,0x75,0xdf,0x6e + .byte 0x47,0xf1,0x1a,0x71,0x1d,0x29,0xc5,0x89 + .byte 0x6f,0xb7,0x62,0x0e,0xaa,0x18,0xbe,0x1b + .byte 0xfc,0x56,0x3e,0x4b,0xc6,0xd2,0x79,0x20 + .byte 0x9a,0xdb,0xc0,0xfe,0x78,0xcd,0x5a,0xf4 + .byte 0x1f,0xdd,0xa8,0x33,0x88,0x07,0xc7,0x31 + .byte 0xb1,0x12,0x10,0x59,0x27,0x80,0xec,0x5f + .byte 0x60,0x51,0x7f,0xa9,0x19,0xb5,0x4a,0x0d + .byte 0x2d,0xe5,0x7a,0x9f,0x93,0xc9,0x9c,0xef + .byte 0xa0,0xe0,0x3b,0x4d,0xae,0x2a,0xf5,0xb0 + .byte 0xc8,0xeb,0xbb,0x3c,0x83,0x53,0x99,0x61 + .byte 0x17,0x2b,0x04,0x7e,0xba,0x77,0xd6,0x26 + .byte 0xe1,0x69,0x14,0x63,0x55,0x21,0x0c,0x7d + .text + .align 4 + .globl AES_CBC_decrypt + .type AES_CBC_decrypt, %function +AES_CBC_decrypt: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + mov lr, r0 + adr r0, L_AES_ARM32_td_ecb + ldr r0, [r0] + mov r12, r2 + adr r2, L_AES_ARM32_cbc_td4 + ldr r8, [sp, #36] + ldr r4, [sp, #40] + push {r3-r4} + cmp r8, #10 + beq L_AES_CBC_decrypt_loop_block_128 + cmp r8, #12 + beq L_AES_CBC_decrypt_loop_block_192 +L_AES_CBC_decrypt_loop_block_256: + push {r1, r12, lr} ldr r4, [lr] ldr r5, [lr, #4] ldr r6, [lr, #8] ldr r7, [lr, #12] + ldr lr, [sp, #16] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + str r4, [lr, #16] + str r5, [lr, #20] +#else + strd r4, r5, [lr, #16] +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + str r6, [lr, #24] + str r7, [lr, #28] +#else + strd r6, r7, [lr, #24] +#endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) eor r8, r4, r4, ror #16 eor r9, r5, r5, ror #16 @@ -18366,18 +11050,17 @@ L_AES_ECB_decrypt_loop_block_128: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - push {r1, r3, r12, lr} ldm r3!, {r8, r9, r10, r11} # Round: 0 - XOR in key schedule eor r4, r4, r8 eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #4 + mov r1, #6 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_decrypt_block #else -L_AES_ECB_decrypt_block_nr_128: +L_AES_CBC_decrypt_block_nr_256_odd: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -18695,7 +11378,7 @@ L_AES_ECB_decrypt_block_nr_128: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_ECB_decrypt_block_nr_128 + bne L_AES_CBC_decrypt_block_nr_256_odd #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -19002,119 +11685,56 @@ L_AES_ECB_decrypt_block_nr_128: ldrb r11, [r2, r11] ldrb r12, [r2, r12] ldrb r7, [r2, r7] - ldrb lr, [r2, lr] - eor r12, r12, r11, lsl #16 - ldm r3, {r8, r9, r10, r11} - eor r7, r7, r12, lsl #8 - eor r7, r7, lr, lsl #16 - # XOR in Key Schedule - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 -#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - pop {r1, r3, r12, lr} -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r8, r4, r4, ror #16 - eor r9, r5, r5, ror #16 - eor r10, r6, r6, ror #16 - eor r11, r7, r7, ror #16 - bic r8, r8, #0xff0000 - bic r9, r9, #0xff0000 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - ror r6, r6, #8 - ror r7, r7, #8 - eor r4, r4, r8, lsr #8 - eor r5, r5, r9, lsr #8 - eor r6, r6, r10, lsr #8 - eor r7, r7, r11, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - str r4, [r1] - str r5, [r1, #4] - str r6, [r1, #8] - str r7, [r1, #12] - subs r12, r12, #16 - add lr, lr, #16 - add r1, r1, #16 - bne L_AES_ECB_decrypt_loop_block_128 -L_AES_ECB_decrypt_end: - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size AES_ECB_decrypt,.-AES_ECB_decrypt -#endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER || defined(HAVE_AES_ECB) */ -#ifdef HAVE_AES_CBC -#ifndef __APPLE__ - .text - .type L_AES_ARM32_cbc_td4, %object - .size L_AES_ARM32_cbc_td4, 256 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 4-byte aligned, 32-bit aligned -#ifndef __APPLE__ - .align 2 -#else - .p2align 2 -#endif /* __APPLE__ */ -L_AES_ARM32_cbc_td4: - .byte 0x52,0x09,0x6a,0xd5,0x30,0x36,0xa5,0x38 - .byte 0xbf,0x40,0xa3,0x9e,0x81,0xf3,0xd7,0xfb - .byte 0x7c,0xe3,0x39,0x82,0x9b,0x2f,0xff,0x87 - .byte 0x34,0x8e,0x43,0x44,0xc4,0xde,0xe9,0xcb - .byte 0x54,0x7b,0x94,0x32,0xa6,0xc2,0x23,0x3d - .byte 0xee,0x4c,0x95,0x0b,0x42,0xfa,0xc3,0x4e - .byte 0x08,0x2e,0xa1,0x66,0x28,0xd9,0x24,0xb2 - .byte 0x76,0x5b,0xa2,0x49,0x6d,0x8b,0xd1,0x25 - .byte 0x72,0xf8,0xf6,0x64,0x86,0x68,0x98,0x16 - .byte 0xd4,0xa4,0x5c,0xcc,0x5d,0x65,0xb6,0x92 - .byte 0x6c,0x70,0x48,0x50,0xfd,0xed,0xb9,0xda - .byte 0x5e,0x15,0x46,0x57,0xa7,0x8d,0x9d,0x84 - .byte 0x90,0xd8,0xab,0x00,0x8c,0xbc,0xd3,0x0a - .byte 0xf7,0xe4,0x58,0x05,0xb8,0xb3,0x45,0x06 - .byte 0xd0,0x2c,0x1e,0x8f,0xca,0x3f,0x0f,0x02 - .byte 0xc1,0xaf,0xbd,0x03,0x01,0x13,0x8a,0x6b - .byte 0x3a,0x91,0x11,0x41,0x4f,0x67,0xdc,0xea - .byte 0x97,0xf2,0xcf,0xce,0xf0,0xb4,0xe6,0x73 - .byte 0x96,0xac,0x74,0x22,0xe7,0xad,0x35,0x85 - .byte 0xe2,0xf9,0x37,0xe8,0x1c,0x75,0xdf,0x6e - .byte 0x47,0xf1,0x1a,0x71,0x1d,0x29,0xc5,0x89 - .byte 0x6f,0xb7,0x62,0x0e,0xaa,0x18,0xbe,0x1b - .byte 0xfc,0x56,0x3e,0x4b,0xc6,0xd2,0x79,0x20 - .byte 0x9a,0xdb,0xc0,0xfe,0x78,0xcd,0x5a,0xf4 - .byte 0x1f,0xdd,0xa8,0x33,0x88,0x07,0xc7,0x31 - .byte 0xb1,0x12,0x10,0x59,0x27,0x80,0xec,0x5f - .byte 0x60,0x51,0x7f,0xa9,0x19,0xb5,0x4a,0x0d - .byte 0x2d,0xe5,0x7a,0x9f,0x93,0xc9,0x9c,0xef - .byte 0xa0,0xe0,0x3b,0x4d,0xae,0x2a,0xf5,0xb0 - .byte 0xc8,0xeb,0xbb,0x3c,0x83,0x53,0x99,0x61 - .byte 0x17,0x2b,0x04,0x7e,0xba,0x77,0xd6,0x26 - .byte 0xe1,0x69,0x14,0x63,0x55,0x21,0x0c,0x7d - .text - .align 4 - .globl AES_CBC_decrypt - .type AES_CBC_decrypt, %function -AES_CBC_decrypt: - push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - mov lr, r0 - adr r0, L_AES_ARM32_td_ecb - ldr r0, [r0] - mov r12, r2 - adr r2, L_AES_ARM32_cbc_td4 - ldr r8, [sp, #36] - ldr r4, [sp, #40] - push {r3-r4} - cmp r8, #10 - beq L_AES_CBC_decrypt_loop_block_128 - cmp r8, #12 - beq L_AES_CBC_decrypt_loop_block_192 -L_AES_CBC_decrypt_loop_block_256: + ldrb lr, [r2, lr] + eor r12, r12, r11, lsl #16 + ldm r3, {r8, r9, r10, r11} + eor r7, r7, r12, lsl #8 + eor r7, r7, lr, lsl #16 + # XOR in Key Schedule + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 +#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ + ldr lr, [sp, #16] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + eor r8, r4, r4, ror #16 + eor r9, r5, r5, ror #16 + eor r10, r6, r6, ror #16 + eor r11, r7, r7, ror #16 + bic r8, r8, #0xff0000 + bic r9, r9, #0xff0000 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 + ror r6, r6, #8 + ror r7, r7, #8 + eor r4, r4, r8, lsr #8 + eor r5, r5, r9, lsr #8 + eor r6, r6, r10, lsr #8 + eor r7, r7, r11, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + ldm lr, {r8, r9, r10, r11} + pop {r1, r12, lr} + ldr r3, [sp] + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + str r4, [r1] + str r5, [r1, #4] + str r6, [r1, #8] + str r7, [r1, #12] + subs r12, r12, #16 + add lr, lr, #16 + add r1, r1, #16 + beq L_AES_CBC_decrypt_end_odd push {r1, r12, lr} ldr r4, [lr] ldr r5, [lr, #4] @@ -19122,16 +11742,15 @@ L_AES_CBC_decrypt_loop_block_256: ldr r7, [lr, #12] ldr lr, [sp, #16] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - str r4, [lr, #16] - str r5, [lr, #20] + stm lr, {r4, r5} #else - strd r4, r5, [lr, #16] + strd r4, r5, [lr] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - str r6, [lr, #24] - str r7, [lr, #28] + str r6, [lr, #8] + str r7, [lr, #12] #else - strd r6, r7, [lr, #24] + strd r6, r7, [lr, #8] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) eor r8, r4, r4, ror #16 @@ -19166,7 +11785,7 @@ L_AES_CBC_decrypt_loop_block_256: #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_decrypt_block #else -L_AES_CBC_decrypt_block_nr_256_odd: +L_AES_CBC_decrypt_block_nr_256_even: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -19484,7 +12103,7 @@ L_AES_CBC_decrypt_block_nr_256_odd: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_CBC_decrypt_block_nr_256_odd + bne L_AES_CBC_decrypt_block_nr_256_even #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -19826,7 +12445,18 @@ L_AES_CBC_decrypt_block_nr_256_odd: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - ldm lr, {r8, r9, r10, r11} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + ldr r8, [lr, #16] + ldr r9, [lr, #20] +#else + ldrd r8, r9, [lr, #16] +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + ldr r10, [lr, #24] + ldr r11, [lr, #28] +#else + ldrd r10, r11, [lr, #24] +#endif pop {r1, r12, lr} ldr r3, [sp] eor r4, r4, r8 @@ -19840,7 +12470,9 @@ L_AES_CBC_decrypt_block_nr_256_odd: subs r12, r12, #16 add lr, lr, #16 add r1, r1, #16 - beq L_AES_CBC_decrypt_end_odd + bne L_AES_CBC_decrypt_loop_block_256 + b L_AES_CBC_decrypt_end +L_AES_CBC_decrypt_loop_block_192: push {r1, r12, lr} ldr r4, [lr] ldr r5, [lr, #4] @@ -19848,15 +12480,16 @@ L_AES_CBC_decrypt_block_nr_256_odd: ldr r7, [lr, #12] ldr lr, [sp, #16] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm lr, {r4, r5} + str r4, [lr, #16] + str r5, [lr, #20] #else - strd r4, r5, [lr] + strd r4, r5, [lr, #16] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - str r6, [lr, #8] - str r7, [lr, #12] + str r6, [lr, #24] + str r7, [lr, #28] #else - strd r6, r7, [lr, #8] + strd r6, r7, [lr, #24] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) eor r8, r4, r4, ror #16 @@ -19887,11 +12520,11 @@ L_AES_CBC_decrypt_block_nr_256_odd: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #6 + mov r1, #5 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_decrypt_block #else -L_AES_CBC_decrypt_block_nr_256_even: +L_AES_CBC_decrypt_block_nr_192_odd: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -20209,7 +12842,7 @@ L_AES_CBC_decrypt_block_nr_256_even: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_CBC_decrypt_block_nr_256_even + bne L_AES_CBC_decrypt_block_nr_192_odd #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -20551,18 +13184,7 @@ L_AES_CBC_decrypt_block_nr_256_even: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - ldr r8, [lr, #16] - ldr r9, [lr, #20] -#else - ldrd r8, r9, [lr, #16] -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - ldr r10, [lr, #24] - ldr r11, [lr, #28] -#else - ldrd r10, r11, [lr, #24] -#endif + ldm lr, {r8, r9, r10, r11} pop {r1, r12, lr} ldr r3, [sp] eor r4, r4, r8 @@ -20576,9 +13198,7 @@ L_AES_CBC_decrypt_block_nr_256_even: subs r12, r12, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_CBC_decrypt_loop_block_256 - b L_AES_CBC_decrypt_end -L_AES_CBC_decrypt_loop_block_192: + beq L_AES_CBC_decrypt_end_odd push {r1, r12, lr} ldr r4, [lr] ldr r5, [lr, #4] @@ -20586,16 +13206,15 @@ L_AES_CBC_decrypt_loop_block_192: ldr r7, [lr, #12] ldr lr, [sp, #16] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - str r4, [lr, #16] - str r5, [lr, #20] + stm lr, {r4, r5} #else - strd r4, r5, [lr, #16] + strd r4, r5, [lr] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - str r6, [lr, #24] - str r7, [lr, #28] + str r6, [lr, #8] + str r7, [lr, #12] #else - strd r6, r7, [lr, #24] + strd r6, r7, [lr, #8] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) eor r8, r4, r4, ror #16 @@ -20630,7 +13249,7 @@ L_AES_CBC_decrypt_loop_block_192: #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_decrypt_block #else -L_AES_CBC_decrypt_block_nr_192_odd: +L_AES_CBC_decrypt_block_nr_192_even: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -20948,7 +13567,7 @@ L_AES_CBC_decrypt_block_nr_192_odd: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_CBC_decrypt_block_nr_192_odd + bne L_AES_CBC_decrypt_block_nr_192_even #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -21290,7 +13909,18 @@ L_AES_CBC_decrypt_block_nr_192_odd: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - ldm lr, {r8, r9, r10, r11} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + ldr r8, [lr, #16] + ldr r9, [lr, #20] +#else + ldrd r8, r9, [lr, #16] +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + ldr r10, [lr, #24] + ldr r11, [lr, #28] +#else + ldrd r10, r11, [lr, #24] +#endif pop {r1, r12, lr} ldr r3, [sp] eor r4, r4, r8 @@ -21304,7 +13934,9 @@ L_AES_CBC_decrypt_block_nr_192_odd: subs r12, r12, #16 add lr, lr, #16 add r1, r1, #16 - beq L_AES_CBC_decrypt_end_odd + bne L_AES_CBC_decrypt_loop_block_192 + b L_AES_CBC_decrypt_end +L_AES_CBC_decrypt_loop_block_128: push {r1, r12, lr} ldr r4, [lr] ldr r5, [lr, #4] @@ -21312,15 +13944,16 @@ L_AES_CBC_decrypt_block_nr_192_odd: ldr r7, [lr, #12] ldr lr, [sp, #16] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm lr, {r4, r5} + str r4, [lr, #16] + str r5, [lr, #20] #else - strd r4, r5, [lr] + strd r4, r5, [lr, #16] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - str r6, [lr, #8] - str r7, [lr, #12] + str r6, [lr, #24] + str r7, [lr, #28] #else - strd r6, r7, [lr, #8] + strd r6, r7, [lr, #24] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) eor r8, r4, r4, ror #16 @@ -21351,11 +13984,11 @@ L_AES_CBC_decrypt_block_nr_192_odd: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #5 + mov r1, #4 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_decrypt_block #else -L_AES_CBC_decrypt_block_nr_192_even: +L_AES_CBC_decrypt_block_nr_128_odd: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -21673,7 +14306,7 @@ L_AES_CBC_decrypt_block_nr_192_even: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_CBC_decrypt_block_nr_192_even + bne L_AES_CBC_decrypt_block_nr_128_odd #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -22015,18 +14648,7 @@ L_AES_CBC_decrypt_block_nr_192_even: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - ldr r8, [lr, #16] - ldr r9, [lr, #20] -#else - ldrd r8, r9, [lr, #16] -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - ldr r10, [lr, #24] - ldr r11, [lr, #28] -#else - ldrd r10, r11, [lr, #24] -#endif + ldm lr, {r8, r9, r10, r11} pop {r1, r12, lr} ldr r3, [sp] eor r4, r4, r8 @@ -22040,9 +14662,7 @@ L_AES_CBC_decrypt_block_nr_192_even: subs r12, r12, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_CBC_decrypt_loop_block_192 - b L_AES_CBC_decrypt_end -L_AES_CBC_decrypt_loop_block_128: + beq L_AES_CBC_decrypt_end_odd push {r1, r12, lr} ldr r4, [lr] ldr r5, [lr, #4] @@ -22050,16 +14670,15 @@ L_AES_CBC_decrypt_loop_block_128: ldr r7, [lr, #12] ldr lr, [sp, #16] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - str r4, [lr, #16] - str r5, [lr, #20] + stm lr, {r4, r5} #else - strd r4, r5, [lr, #16] + strd r4, r5, [lr] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - str r6, [lr, #24] - str r7, [lr, #28] + str r6, [lr, #8] + str r7, [lr, #12] #else - strd r6, r7, [lr, #24] + strd r6, r7, [lr, #8] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) eor r8, r4, r4, ror #16 @@ -22094,7 +14713,7 @@ L_AES_CBC_decrypt_loop_block_128: #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_decrypt_block #else -L_AES_CBC_decrypt_block_nr_128_odd: +L_AES_CBC_decrypt_block_nr_128_even: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -22412,7 +15031,7 @@ L_AES_CBC_decrypt_block_nr_128_odd: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_CBC_decrypt_block_nr_128_odd + bne L_AES_CBC_decrypt_block_nr_128_even #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r7, #8 @@ -22754,543 +15373,1000 @@ L_AES_CBC_decrypt_block_nr_128_odd: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - ldm lr, {r8, r9, r10, r11} - pop {r1, r12, lr} - ldr r3, [sp] - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 - str r4, [r1] - str r5, [r1, #4] - str r6, [r1, #8] - str r7, [r1, #12] - subs r12, r12, #16 - add lr, lr, #16 - add r1, r1, #16 - beq L_AES_CBC_decrypt_end_odd - push {r1, r12, lr} - ldr r4, [lr] - ldr r5, [lr, #4] - ldr r6, [lr, #8] - ldr r7, [lr, #12] - ldr lr, [sp, #16] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm lr, {r4, r5} + ldr r8, [lr, #16] + ldr r9, [lr, #20] #else - strd r4, r5, [lr] + ldrd r8, r9, [lr, #16] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - str r6, [lr, #8] - str r7, [lr, #12] + ldr r10, [lr, #24] + ldr r11, [lr, #28] #else - strd r6, r7, [lr, #8] + ldrd r10, r11, [lr, #24] #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r8, r4, r4, ror #16 - eor r9, r5, r5, ror #16 - eor r10, r6, r6, ror #16 - eor r11, r7, r7, ror #16 - bic r8, r8, #0xff0000 - bic r9, r9, #0xff0000 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - ror r6, r6, #8 - ror r7, r7, #8 - eor r4, r4, r8, lsr #8 - eor r5, r5, r9, lsr #8 - eor r6, r6, r10, lsr #8 - eor r7, r7, r11, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - ldm r3!, {r8, r9, r10, r11} - # Round: 0 - XOR in key schedule + pop {r1, r12, lr} + ldr r3, [sp] eor r4, r4, r8 eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #4 -#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE - bl AES_decrypt_block -#else -L_AES_CBC_decrypt_block_nr_128_even: + str r4, [r1] + str r5, [r1, #4] + str r6, [r1, #8] + str r7, [r1, #12] + subs r12, r12, #16 + add lr, lr, #16 + add r1, r1, #16 + bne L_AES_CBC_decrypt_loop_block_128 + b L_AES_CBC_decrypt_end +L_AES_CBC_decrypt_end_odd: + ldr r4, [sp, #4] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r7, #8 - lsr r8, r8, #24 -#else - uxtb r8, r7, ror #16 -#endif + ldr r8, [r4, #16] + ldr r9, [r4, #20] #else - ubfx r8, r7, #16, #8 + ldrd r8, r9, [r4, #16] #endif - lsr r11, r4, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r6, #16 - lsr r12, r12, #24 + ldr r10, [r4, #24] + ldr r11, [r4, #28] #else - uxtb r12, r6, ror #8 + ldrd r10, r11, [r4, #24] #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r4, {r8, r9} #else - ubfx r12, r6, #8, #8 + strd r8, r9, [r4] #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r5, #24 - lsr lr, lr, #24 + str r10, [r4, #8] + str r11, [r4, #12] #else - uxtb lr, r5 + strd r10, r11, [r4, #8] #endif +L_AES_CBC_decrypt_end: + pop {r3-r4} + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size AES_CBC_decrypt,.-AES_CBC_decrypt +#endif /* HAVE_AES_CBC */ +#endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER || HAVE_AES_CBC + * HAVE_AES_ECB */ +#endif /* HAVE_AES_DECRYPT */ +#ifdef HAVE_AESGCM +#ifndef __APPLE__ + .text + .type L_GCM_gmult_len_r, %object + .size L_GCM_gmult_len_r, 64 #else - ubfx lr, r5, #0, #8 + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_GCM_gmult_len_r: + .long 0x00000000,0x1c200000,0x38400000,0x24600000 + .long 0x70800000,0x6ca00000,0x48c00000,0x54e00000 + .long 0xe1000000,0xfd200000,0xd9400000,0xc5600000 + .long 0x91800000,0x8da00000,0xa9c00000,0xb5e00000 + .text + .align 4 + .globl GCM_gmult_len + .type GCM_gmult_len, %function +GCM_gmult_len: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + adr lr, L_GCM_gmult_len_r +L_GCM_gmult_len_start_block: + push {r3} + ldr r12, [r0, #12] + ldr r3, [r2, #12] + eor r12, r12, r3 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r3, r12, #4 + lsr r3, r3, #28 +#else + ubfx r3, r12, #24, #4 #endif - ldr r8, [r0, r8, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr r12, [r0, r12, lsl #2] - ldr lr, [r0, lr, lsl #2] + add r3, r1, r3, lsl #4 + ldm r3, {r8, r9, r10, r11} + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r9, r4, #8 - lsr r9, r9, #24 + lsr r4, r12, #28 #else - uxtb r9, r4, ror #16 + ubfx r4, r12, #28, #4 #endif + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r4, r12, #12 + lsr r4, r4, #28 #else - ubfx r9, r4, #16, #8 + ubfx r4, r12, #16, #4 #endif - eor r8, r8, r11, ror #24 - lsr r11, r5, #24 - eor r8, r8, r12, ror #8 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r7, #16 - lsr r12, r12, #24 + lsl r4, r12, #8 + lsr r4, r4, #28 #else - uxtb r12, r7, ror #8 + ubfx r4, r12, #20, #4 #endif + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r4, r12, #20 + lsr r4, r4, #28 #else - ubfx r12, r7, #8, #8 + ubfx r4, r12, #8, #4 #endif - eor r8, r8, lr, ror #16 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r6, #24 - lsr lr, lr, #24 + lsl r4, r12, #16 + lsr r4, r4, #28 #else - uxtb lr, r6 + ubfx r4, r12, #12, #4 #endif + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 + and r4, r12, #15 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r4, r12, #24 + lsr r4, r4, #28 #else - ubfx lr, r6, #0, #8 + ubfx r4, r12, #4, #4 #endif - ldr r9, [r0, r9, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr r12, [r0, r12, lsl #2] - ldr lr, [r0, lr, lsl #2] + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + ldr r12, [r0, #8] + ldr r3, [r2, #8] + eor r12, r12, r3 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r10, r5, #8 - lsr r10, r10, #24 -#else - uxtb r10, r5, ror #16 -#endif + lsl r3, r12, #4 + lsr r3, r3, #28 #else - ubfx r10, r5, #16, #8 + ubfx r3, r12, #24, #4 #endif - eor r9, r9, r11, ror #24 - lsr r11, r6, #24 - eor r9, r9, r12, ror #8 + add r3, r1, r3, lsl #4 + ldm r3, {r4, r5, r6, r7} + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r4, #16 - lsr r12, r12, #24 + lsr r4, r12, #28 #else - uxtb r12, r4, ror #8 + ubfx r4, r12, #28, #4 #endif + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r4, r12, #12 + lsr r4, r4, #28 #else - ubfx r12, r4, #8, #8 + ubfx r4, r12, #16, #4 #endif - eor r9, r9, lr, ror #16 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r7, #24 - lsr lr, lr, #24 + lsl r4, r12, #8 + lsr r4, r4, #28 #else - uxtb lr, r7 + ubfx r4, r12, #20, #4 #endif + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r4, r12, #20 + lsr r4, r4, #28 #else - ubfx lr, r7, #0, #8 + ubfx r4, r12, #8, #4 #endif - ldr r10, [r0, r10, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr r12, [r0, r12, lsl #2] - ldr lr, [r0, lr, lsl #2] + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r4, #24 - lsr r4, r4, #24 + lsl r4, r12, #16 + lsr r4, r4, #28 #else - uxtb r4, r4 + ubfx r4, r12, #12, #4 #endif + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 + and r4, r12, #15 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r4, r12, #24 + lsr r4, r4, #28 #else - ubfx r4, r4, #0, #8 + ubfx r4, r12, #4, #4 #endif - eor r10, r10, r11, ror #24 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + ldr r12, [r0, #4] + ldr r3, [r2, #4] + eor r12, r12, r3 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r11, r6, #8 - lsr r11, r11, #24 + lsl r3, r12, #4 + lsr r3, r3, #28 #else - uxtb r11, r6, ror #16 + ubfx r3, r12, #24, #4 #endif + add r3, r1, r3, lsl #4 + ldm r3, {r4, r5, r6, r7} + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsr r4, r12, #28 #else - ubfx r11, r6, #16, #8 + ubfx r4, r12, #28, #4 #endif - eor r10, r10, r12, ror #8 - lsr r12, r7, #24 - eor r10, r10, lr, ror #16 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r5, #16 - lsr lr, lr, #24 + lsl r4, r12, #12 + lsr r4, r4, #28 #else - uxtb lr, r5, ror #8 + ubfx r4, r12, #16, #4 #endif + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r4, r12, #8 + lsr r4, r4, #28 #else - ubfx lr, r5, #8, #8 + ubfx r4, r12, #20, #4 #endif - ldr r4, [r0, r4, lsl #2] - ldr r12, [r0, r12, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr lr, [r0, lr, lsl #2] - eor r12, r12, r4, ror #24 - ldm r3!, {r4, r5, r6, r7} - eor r11, r11, lr, ror #8 - eor r11, r11, r12, ror #24 - # XOR in Key Schedule + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 eor r8, r8, r4 eor r9, r9, r5 eor r10, r10, r6 eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r11, #8 - lsr r4, r4, #24 -#else - uxtb r4, r11, ror #16 -#endif -#else - ubfx r4, r11, #16, #8 -#endif - lsr r7, r8, #24 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r10, #16 - lsr r12, r12, #24 -#else - uxtb r12, r10, ror #8 -#endif + lsl r4, r12, #20 + lsr r4, r4, #28 #else - ubfx r12, r10, #8, #8 + ubfx r4, r12, #8, #4 #endif + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r9, #24 - lsr lr, lr, #24 -#else - uxtb lr, r9 -#endif + lsl r4, r12, #16 + lsr r4, r4, #28 #else - ubfx lr, r9, #0, #8 + ubfx r4, r12, #12, #4 #endif - ldr r4, [r0, r4, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr r12, [r0, r12, lsl #2] - ldr lr, [r0, lr, lsl #2] + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 + and r4, r12, #15 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r8, #8 - lsr r5, r5, #24 -#else - uxtb r5, r8, ror #16 -#endif + lsl r4, r12, #24 + lsr r4, r4, #28 #else - ubfx r5, r8, #16, #8 + ubfx r4, r12, #4, #4 #endif - eor r4, r4, r7, ror #24 - lsr r7, r9, #24 - eor r4, r4, r12, ror #8 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + ldr r12, [r0] + ldr r3, [r2] + eor r12, r12, r3 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r11, #16 - lsr r12, r12, #24 -#else - uxtb r12, r11, ror #8 -#endif + lsl r3, r12, #4 + lsr r3, r3, #28 #else - ubfx r12, r11, #8, #8 + ubfx r3, r12, #24, #4 #endif - eor r4, r4, lr, ror #16 + add r3, r1, r3, lsl #4 + ldm r3, {r4, r5, r6, r7} + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r10, #24 - lsr lr, lr, #24 -#else - uxtb lr, r10 -#endif + lsr r4, r12, #28 #else - ubfx lr, r10, #0, #8 + ubfx r4, r12, #28, #4 #endif - ldr r5, [r0, r5, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr r12, [r0, r12, lsl #2] - ldr lr, [r0, lr, lsl #2] + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r9, #8 - lsr r6, r6, #24 -#else - uxtb r6, r9, ror #16 -#endif + lsl r4, r12, #12 + lsr r4, r4, #28 #else - ubfx r6, r9, #16, #8 + ubfx r4, r12, #16, #4 #endif - eor r5, r5, r7, ror #24 - lsr r7, r10, #24 - eor r5, r5, r12, ror #8 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r8, #16 - lsr r12, r12, #24 -#else - uxtb r12, r8, ror #8 -#endif + lsl r4, r12, #8 + lsr r4, r4, #28 #else - ubfx r12, r8, #8, #8 + ubfx r4, r12, #20, #4 #endif - eor r5, r5, lr, ror #16 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r11, #24 - lsr lr, lr, #24 -#else - uxtb lr, r11 -#endif + lsl r4, r12, #20 + lsr r4, r4, #28 #else - ubfx lr, r11, #0, #8 + ubfx r4, r12, #8, #4 #endif - ldr r6, [r0, r6, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr r12, [r0, r12, lsl #2] - ldr lr, [r0, lr, lsl #2] + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r8, #24 - lsr r8, r8, #24 + lsl r4, r12, #16 + lsr r4, r4, #28 #else - uxtb r8, r8 + ubfx r4, r12, #12, #4 #endif + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 + and r4, r12, #15 + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + lsr r6, r10, #4 + and r3, r11, #15 + lsr r11, r11, #4 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r4, r12, #24 + lsr r4, r4, #28 #else - ubfx r8, r8, #0, #8 + ubfx r4, r12, #4, #4 #endif - eor r6, r6, r7, ror #24 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + eor r11, r11, r10, lsl #28 + ldr r3, [lr, r3, lsl #2] + add r4, r1, r4, lsl #4 + eor r10, r6, r9, lsl #28 + lsr r9, r9, #4 + ldm r4, {r4, r5, r6, r7} + eor r9, r9, r8, lsl #28 + eor r8, r3, r8, lsr #4 + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r10, #8 - lsr r7, r7, #24 + # REV r8, r8 + eor r3, r8, r8, ror #16 + bic r3, r3, #0xff0000 + ror r8, r8, #8 + eor r8, r8, r3, lsr #8 + # REV r9, r9 + eor r3, r9, r9, ror #16 + bic r3, r3, #0xff0000 + ror r9, r9, #8 + eor r9, r9, r3, lsr #8 + # REV r10, r10 + eor r3, r10, r10, ror #16 + bic r3, r3, #0xff0000 + ror r10, r10, #8 + eor r10, r10, r3, lsr #8 + # REV r11, r11 + eor r3, r11, r11, ror #16 + bic r3, r3, #0xff0000 + ror r11, r11, #8 + eor r11, r11, r3, lsr #8 #else - uxtb r7, r10, ror #16 -#endif + rev r8, r8 + rev r9, r9 + rev r10, r10 + rev r11, r11 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r0, {r8, r9, r10, r11} + pop {r3} + subs r3, r3, #16 + add r2, r2, #16 + bne L_GCM_gmult_len_start_block + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size GCM_gmult_len,.-GCM_gmult_len +#ifndef __APPLE__ + .text + .type L_AES_ARM32_te_gcm, %object + .size L_AES_ARM32_te_gcm, 12 #else - ubfx r7, r10, #16, #8 -#endif - eor r6, r6, r12, ror #8 - lsr r12, r11, #24 - eor r6, r6, lr, ror #16 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r9, #16 - lsr lr, lr, #24 + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 #else - uxtb lr, r9, ror #8 -#endif + .p2align 3 +#endif /* __APPLE__ */ +L_AES_ARM32_te_gcm: + .long L_AES_ARM32_te_data + .text + .align 4 + .globl AES_GCM_encrypt + .type AES_GCM_encrypt, %function +AES_GCM_encrypt: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + ldr r12, [sp, #36] + ldr r8, [sp, #40] + mov lr, r0 + adr r0, L_AES_ARM32_te_gcm + ldr r0, [r0] + ldm r8, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + eor r10, r4, r4, ror #16 + eor r11, r5, r5, ror #16 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 + eor r4, r4, r10, lsr #8 + eor r5, r5, r11, lsr #8 + eor r10, r6, r6, ror #16 + eor r11, r7, r7, ror #16 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r6, r6, #8 + ror r7, r7, #8 + eor r6, r6, r10, lsr #8 + eor r7, r7, r11, lsr #8 #else - ubfx lr, r9, #8, #8 -#endif - ldr r8, [r0, r8, lsl #2] - ldr r12, [r0, r12, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr lr, [r0, lr, lsl #2] - eor r12, r12, r8, ror #24 + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r8, {r4, r5, r6, r7} + push {r3, r8} + cmp r12, #10 + beq L_AES_GCM_encrypt_start_block_128 + cmp r12, #12 + beq L_AES_GCM_encrypt_start_block_192 +L_AES_GCM_encrypt_loop_block_256: + push {r1, r2, lr} + ldr lr, [sp, #16] + add r7, r7, #1 ldm r3!, {r8, r9, r10, r11} - eor r7, r7, lr, ror #8 - eor r7, r7, r12, ror #24 - # XOR in Key Schedule + str r7, [lr, #12] + # Round: 0 - XOR in key schedule eor r4, r4, r8 eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - subs r1, r1, #1 - bne L_AES_CBC_decrypt_block_nr_128_even + mov r1, #6 +#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE + bl AES_encrypt_block +#else +L_AES_GCM_encrypt_block_nr_256: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r7, #8 + lsl r8, r5, #8 lsr r8, r8, #24 #else - uxtb r8, r7, ror #16 + uxtb r8, r5, ror #16 #endif #else - ubfx r8, r7, #16, #8 + ubfx r8, r5, #16, #8 #endif lsr r11, r4, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r6, #16 - lsr r12, r12, #24 + lsl lr, r6, #16 + lsr lr, lr, #24 #else - uxtb r12, r6, ror #8 + uxtb lr, r6, ror #8 #endif #else - ubfx r12, r6, #8, #8 + ubfx lr, r6, #8, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r5, #24 - lsr lr, lr, #24 + lsl r2, r7, #24 + lsr r2, r2, #24 #else - uxtb lr, r5 + uxtb r2, r7 #endif #else - ubfx lr, r5, #0, #8 + ubfx r2, r7, #0, #8 #endif ldr r8, [r0, r8, lsl #2] ldr r11, [r0, r11, lsl #2] - ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r9, r4, #8 + lsl r9, r6, #8 lsr r9, r9, #24 #else - uxtb r9, r4, ror #16 + uxtb r9, r6, ror #16 #endif #else - ubfx r9, r4, #16, #8 + ubfx r9, r6, #16, #8 #endif eor r8, r8, r11, ror #24 lsr r11, r5, #24 - eor r8, r8, r12, ror #8 + eor r8, r8, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r7, #16 - lsr r12, r12, #24 + lsl lr, r7, #16 + lsr lr, lr, #24 #else - uxtb r12, r7, ror #8 + uxtb lr, r7, ror #8 #endif #else - ubfx r12, r7, #8, #8 + ubfx lr, r7, #8, #8 #endif - eor r8, r8, lr, ror #16 + eor r8, r8, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r6, #24 - lsr lr, lr, #24 + lsl r2, r4, #24 + lsr r2, r2, #24 #else - uxtb lr, r6 + uxtb r2, r4 #endif #else - ubfx lr, r6, #0, #8 + ubfx r2, r4, #0, #8 #endif ldr r9, [r0, r9, lsl #2] ldr r11, [r0, r11, lsl #2] - ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r10, r5, #8 + lsl r10, r7, #8 lsr r10, r10, #24 #else - uxtb r10, r5, ror #16 + uxtb r10, r7, ror #16 #endif #else - ubfx r10, r5, #16, #8 + ubfx r10, r7, #16, #8 #endif eor r9, r9, r11, ror #24 lsr r11, r6, #24 - eor r9, r9, r12, ror #8 + eor r9, r9, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r4, #16 - lsr r12, r12, #24 + lsl lr, r4, #16 + lsr lr, lr, #24 #else - uxtb r12, r4, ror #8 + uxtb lr, r4, ror #8 #endif #else - ubfx r12, r4, #8, #8 + ubfx lr, r4, #8, #8 #endif - eor r9, r9, lr, ror #16 + eor r9, r9, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r7, #24 - lsr lr, lr, #24 + lsl r2, r5, #24 + lsr r2, r2, #24 #else - uxtb lr, r7 + uxtb r2, r5 #endif #else - ubfx lr, r7, #0, #8 + ubfx r2, r5, #0, #8 #endif ldr r10, [r0, r10, lsl #2] ldr r11, [r0, r11, lsl #2] - ldr r12, [r0, r12, lsl #2] ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r4, #24 - lsr r4, r4, #24 + lsl r6, r6, #24 + lsr r6, r6, #24 #else - uxtb r4, r4 + uxtb r6, r6 #endif #else - ubfx r4, r4, #0, #8 + ubfx r6, r6, #0, #8 #endif eor r10, r10, r11, ror #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r11, r6, #8 + lsl r11, r4, #8 lsr r11, r11, #24 #else - uxtb r11, r6, ror #16 + uxtb r11, r4, ror #16 #endif #else - ubfx r11, r6, #16, #8 + ubfx r11, r4, #16, #8 #endif - eor r10, r10, r12, ror #8 - lsr r12, r7, #24 - eor r10, r10, lr, ror #16 + eor r10, r10, lr, ror #8 + lsr lr, r7, #24 + eor r10, r10, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r5, #16 - lsr lr, lr, #24 + lsl r2, r5, #16 + lsr r2, r2, #24 #else - uxtb lr, r5, ror #8 + uxtb r2, r5, ror #8 #endif #else - ubfx lr, r5, #8, #8 + ubfx r2, r5, #8, #8 #endif - ldr r4, [r0, r4, lsl #2] - ldr r12, [r0, r12, lsl #2] - ldr r11, [r0, r11, lsl #2] + ldr r6, [r0, r6, lsl #2] ldr lr, [r0, lr, lsl #2] - eor r12, r12, r4, ror #24 + ldr r11, [r0, r11, lsl #2] + ldr r2, [r0, r2, lsl #2] + eor lr, lr, r6, ror #24 ldm r3!, {r4, r5, r6, r7} - eor r11, r11, lr, ror #8 - eor r11, r11, r12, ror #24 + eor r11, r11, lr, ror #24 + eor r11, r11, r2, ror #8 # XOR in Key Schedule eor r8, r8, r4 eor r9, r9, r5 @@ -23298,998 +16374,498 @@ L_AES_CBC_decrypt_block_nr_128_even: eor r11, r11, r7 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r9, #24 + lsl r4, r9, #8 lsr r4, r4, #24 #else - uxtb r4, r9 + uxtb r4, r9, ror #16 #endif #else - ubfx r4, r9, #0, #8 + ubfx r4, r9, #16, #8 #endif + lsr r7, r8, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r10, #16 - lsr r7, r7, #24 + lsl lr, r10, #16 + lsr lr, lr, #24 #else - uxtb r7, r10, ror #8 + uxtb lr, r10, ror #8 #endif #else - ubfx r7, r10, #8, #8 + ubfx lr, r10, #8, #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r11, #8 - lsr r12, r12, #24 + lsl r2, r11, #24 + lsr r2, r2, #24 #else - uxtb r12, r11, ror #16 + uxtb r2, r11 #endif #else - ubfx r12, r11, #16, #8 + ubfx r2, r11, #0, #8 #endif - lsr lr, r8, #24 - ldrb r4, [r2, r4] - ldrb r7, [r2, r7] - ldrb r12, [r2, r12] - ldrb lr, [r2, lr] + ldr r4, [r0, r4, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r10, #24 + lsl r5, r10, #8 lsr r5, r5, #24 #else - uxtb r5, r10 + uxtb r5, r10, ror #16 #endif #else - ubfx r5, r10, #0, #8 + ubfx r5, r10, #16, #8 #endif - eor r4, r4, r7, lsl #8 + eor r4, r4, r7, ror #24 + lsr r7, r9, #24 + eor r4, r4, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r11, #16 - lsr r7, r7, #24 + lsl lr, r11, #16 + lsr lr, lr, #24 #else - uxtb r7, r11, ror #8 + uxtb lr, r11, ror #8 #endif #else - ubfx r7, r11, #8, #8 + ubfx lr, r11, #8, #8 #endif - eor r4, r4, r12, lsl #16 + eor r4, r4, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r8, #8 - lsr r12, r12, #24 + lsl r2, r8, #24 + lsr r2, r2, #24 #else - uxtb r12, r8, ror #16 + uxtb r2, r8 #endif #else - ubfx r12, r8, #16, #8 + ubfx r2, r8, #0, #8 #endif - eor r4, r4, lr, lsl #24 - lsr lr, r9, #24 - ldrb r7, [r2, r7] - ldrb lr, [r2, lr] - ldrb r5, [r2, r5] - ldrb r12, [r2, r12] + ldr r5, [r0, r5, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r11, #24 + lsl r6, r11, #8 lsr r6, r6, #24 #else - uxtb r6, r11 -#endif -#else - ubfx r6, r11, #0, #8 -#endif - eor r5, r5, r7, lsl #8 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r8, #16 - lsr r7, r7, #24 -#else - uxtb r7, r8, ror #8 -#endif -#else - ubfx r7, r8, #8, #8 -#endif - eor r5, r5, r12, lsl #16 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r9, #8 - lsr r12, r12, #24 -#else - uxtb r12, r9, ror #16 + uxtb r6, r11, ror #16 #endif #else - ubfx r12, r9, #16, #8 + ubfx r6, r11, #16, #8 #endif - eor r5, r5, lr, lsl #24 - lsr lr, r10, #24 - ldrb r7, [r2, r7] - ldrb lr, [r2, lr] - ldrb r6, [r2, r6] - ldrb r12, [r2, r12] - lsr r11, r11, #24 - eor r6, r6, r7, lsl #8 + eor r5, r5, r7, ror #24 + lsr r7, r10, #24 + eor r5, r5, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r8, #24 - lsr r7, r7, #24 + lsl lr, r8, #16 + lsr lr, lr, #24 #else - uxtb r7, r8 + uxtb lr, r8, ror #8 #endif #else - ubfx r7, r8, #0, #8 + ubfx lr, r8, #8, #8 #endif - eor r6, r6, r12, lsl #16 + eor r5, r5, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r12, r9, #16 - lsr r12, r12, #24 + lsl r2, r9, #24 + lsr r2, r2, #24 #else - uxtb r12, r9, ror #8 + uxtb r2, r9 #endif #else - ubfx r12, r9, #8, #8 + ubfx r2, r9, #0, #8 #endif - eor r6, r6, lr, lsl #24 + ldr r6, [r0, r6, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r10, #8 - lsr lr, lr, #24 -#else - uxtb lr, r10, ror #16 -#endif -#else - ubfx lr, r10, #16, #8 -#endif - ldrb r11, [r2, r11] - ldrb r12, [r2, r12] - ldrb r7, [r2, r7] - ldrb lr, [r2, lr] - eor r12, r12, r11, lsl #16 - ldm r3, {r8, r9, r10, r11} - eor r7, r7, r12, lsl #8 - eor r7, r7, lr, lsl #16 - # XOR in Key Schedule - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 -#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - ldr lr, [sp, #16] -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r8, r4, r4, ror #16 - eor r9, r5, r5, ror #16 - eor r10, r6, r6, ror #16 - eor r11, r7, r7, ror #16 - bic r8, r8, #0xff0000 - bic r9, r9, #0xff0000 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - ror r6, r6, #8 - ror r7, r7, #8 - eor r4, r4, r8, lsr #8 - eor r5, r5, r9, lsr #8 - eor r6, r6, r10, lsr #8 - eor r7, r7, r11, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - ldr r8, [lr, #16] - ldr r9, [lr, #20] -#else - ldrd r8, r9, [lr, #16] -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - ldr r10, [lr, #24] - ldr r11, [lr, #28] + lsl r10, r10, #24 + lsr r10, r10, #24 #else - ldrd r10, r11, [lr, #24] -#endif - pop {r1, r12, lr} - ldr r3, [sp] - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 - str r4, [r1] - str r5, [r1, #4] - str r6, [r1, #8] - str r7, [r1, #12] - subs r12, r12, #16 - add lr, lr, #16 - add r1, r1, #16 - bne L_AES_CBC_decrypt_loop_block_128 - b L_AES_CBC_decrypt_end -L_AES_CBC_decrypt_end_odd: - ldr r4, [sp, #4] -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - ldr r8, [r4, #16] - ldr r9, [r4, #20] + uxtb r10, r10 +#endif #else - ldrd r8, r9, [r4, #16] + ubfx r10, r10, #0, #8 #endif + eor r6, r6, r7, ror #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - ldr r10, [r4, #24] - ldr r11, [r4, #28] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r8, #8 + lsr r7, r7, #24 #else - ldrd r10, r11, [r4, #24] + uxtb r7, r8, ror #16 #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - stm r4, {r8, r9} #else - strd r8, r9, [r4] + ubfx r7, r8, #16, #8 #endif + eor r6, r6, lr, ror #8 + lsr lr, r11, #24 + eor r6, r6, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - str r10, [r4, #8] - str r11, [r4, #12] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r9, #16 + lsr r2, r2, #24 #else - strd r10, r11, [r4, #8] + uxtb r2, r9, ror #8 #endif -L_AES_CBC_decrypt_end: - pop {r3-r4} - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size AES_CBC_decrypt,.-AES_CBC_decrypt -#endif /* HAVE_AES_CBC */ -#endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER || HAVE_AES_CBC - * HAVE_AES_ECB */ -#endif /* HAVE_AES_DECRYPT */ -#ifdef HAVE_AESGCM -#ifndef __APPLE__ - .text - .type L_GCM_gmult_len_r, %object - .size L_GCM_gmult_len_r, 64 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 #else - .p2align 3 -#endif /* __APPLE__ */ -L_GCM_gmult_len_r: - .long 0x00000000,0x1c200000,0x38400000,0x24600000 - .long 0x70800000,0x6ca00000,0x48c00000,0x54e00000 - .long 0xe1000000,0xfd200000,0xd9400000,0xc5600000 - .long 0x91800000,0x8da00000,0xa9c00000,0xb5e00000 - .text - .align 4 - .globl GCM_gmult_len - .type GCM_gmult_len, %function -GCM_gmult_len: - push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - adr lr, L_GCM_gmult_len_r -L_GCM_gmult_len_start_block: - push {r3} - ldr r12, [r0, #12] - ldr r3, [r2, #12] - eor r12, r12, r3 + ubfx r2, r9, #8, #8 +#endif + ldr r10, [r0, r10, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr r2, [r0, r2, lsl #2] + eor lr, lr, r10, ror #24 + ldm r3!, {r8, r9, r10, r11} + eor r7, r7, lr, ror #24 + eor r7, r7, r2, ror #8 + # XOR in Key Schedule + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + subs r1, r1, #1 + bne L_AES_GCM_encrypt_block_nr_256 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r3, r12, #4 - lsr r3, r3, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r8, r5, #8 + lsr r8, r8, #24 #else - ubfx r3, r12, #24, #4 + uxtb r8, r5, ror #16 #endif - add r3, r1, r3, lsl #4 - ldm r3, {r8, r9, r10, r11} - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsr r4, r12, #28 #else - ubfx r4, r12, #28, #4 + ubfx r8, r5, #16, #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 + lsr r11, r4, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #12 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r6, #16 + lsr lr, lr, #24 #else - ubfx r4, r12, #16, #4 + uxtb lr, r6, ror #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #8 - lsr r4, r4, #28 #else - ubfx r4, r12, #20, #4 + ubfx lr, r6, #8, #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #20 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r7, #24 + lsr r2, r2, #24 #else - ubfx r4, r12, #8, #4 + uxtb r2, r7 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #16 - lsr r4, r4, #28 #else - ubfx r4, r12, #12, #4 + ubfx r2, r7, #0, #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 - and r4, r12, #15 - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 + ldr r8, [r0, r8, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #24 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r9, r6, #8 + lsr r9, r9, #24 #else - ubfx r4, r12, #4, #4 + uxtb r9, r6, ror #16 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - ldr r12, [r0, #8] - ldr r3, [r2, #8] - eor r12, r12, r3 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r3, r12, #4 - lsr r3, r3, #28 #else - ubfx r3, r12, #24, #4 + ubfx r9, r6, #16, #8 +#endif + eor r8, r8, r11, ror #24 + lsr r11, r5, #24 + eor r8, r8, lr, ror #8 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r7, #16 + lsr lr, lr, #24 +#else + uxtb lr, r7, ror #8 #endif - add r3, r1, r3, lsl #4 - ldm r3, {r4, r5, r6, r7} - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsr r4, r12, #28 #else - ubfx r4, r12, #28, #4 + ubfx lr, r7, #8, #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 + eor r8, r8, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #12 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r4, #24 + lsr r2, r2, #24 #else - ubfx r4, r12, #16, #4 + uxtb r2, r4 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #8 - lsr r4, r4, #28 #else - ubfx r4, r12, #20, #4 + ubfx r2, r4, #0, #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 + ldr r9, [r0, r9, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #20 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r10, r7, #8 + lsr r10, r10, #24 #else - ubfx r4, r12, #8, #4 + uxtb r10, r7, ror #16 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 +#else + ubfx r10, r7, #16, #8 +#endif + eor r9, r9, r11, ror #24 + lsr r11, r6, #24 + eor r9, r9, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #16 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r4, #16 + lsr lr, lr, #24 #else - ubfx r4, r12, #12, #4 + uxtb lr, r4, ror #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 - and r4, r12, #15 - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 +#else + ubfx lr, r4, #8, #8 +#endif + eor r9, r9, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #24 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r5, #24 + lsr r2, r2, #24 #else - ubfx r4, r12, #4, #4 + uxtb r2, r5 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - ldr r12, [r0, #4] - ldr r3, [r2, #4] - eor r12, r12, r3 +#else + ubfx r2, r5, #0, #8 +#endif + ldr r10, [r0, r10, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r3, r12, #4 - lsr r3, r3, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r6, #24 + lsr r6, r6, #24 #else - ubfx r3, r12, #24, #4 + uxtb r6, r6 #endif - add r3, r1, r3, lsl #4 - ldm r3, {r4, r5, r6, r7} - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 +#else + ubfx r6, r6, #0, #8 +#endif + eor r10, r10, r11, ror #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsr r4, r12, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r11, r4, #8 + lsr r11, r11, #24 #else - ubfx r4, r12, #28, #4 + uxtb r11, r4, ror #16 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 +#else + ubfx r11, r4, #16, #8 +#endif + eor r10, r10, lr, ror #8 + lsr lr, r7, #24 + eor r10, r10, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #12 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r5, #16 + lsr r2, r2, #24 #else - ubfx r4, r12, #16, #4 + uxtb r2, r5, ror #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 +#else + ubfx r2, r5, #8, #8 +#endif + ldr r6, [r0, r6, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr r2, [r0, r2, lsl #2] + eor lr, lr, r6, ror #24 + ldm r3!, {r4, r5, r6, r7} + eor r11, r11, lr, ror #24 + eor r11, r11, r2, ror #8 + # XOR in Key Schedule eor r8, r8, r4 eor r9, r9, r5 eor r10, r10, r6 eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #8 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r4, r11, #24 + lsr r4, r4, #24 #else - ubfx r4, r12, #20, #4 + uxtb r4, r11 +#endif +#else + ubfx r4, r11, #0, #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #20 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r10, #16 + lsr r7, r7, #24 #else - ubfx r4, r12, #8, #4 + uxtb r7, r10, ror #8 +#endif +#else + ubfx r7, r10, #8, #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #16 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r9, #8 + lsr lr, lr, #24 #else - ubfx r4, r12, #12, #4 + uxtb lr, r9, ror #16 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 - and r4, r12, #15 - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 +#else + ubfx lr, r9, #16, #8 +#endif + lsr r2, r8, #24 + ldrb r4, [r0, r4, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #24 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r5, r8, #24 + lsr r5, r5, #24 #else - ubfx r4, r12, #4, #4 + uxtb r5, r8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - ldr r12, [r0] - ldr r3, [r2] - eor r12, r12, r3 +#else + ubfx r5, r8, #0, #8 +#endif + eor r4, r4, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r3, r12, #4 - lsr r3, r3, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r11, #16 + lsr r7, r7, #24 #else - ubfx r3, r12, #24, #4 + uxtb r7, r11, ror #8 #endif - add r3, r1, r3, lsl #4 - ldm r3, {r4, r5, r6, r7} - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 +#else + ubfx r7, r11, #8, #8 +#endif + eor r4, r4, lr, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsr r4, r12, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r10, #8 + lsr lr, lr, #24 #else - ubfx r4, r12, #28, #4 + uxtb lr, r10, ror #16 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 +#else + ubfx lr, r10, #16, #8 +#endif + eor r4, r4, r2, lsl #24 + lsr r2, r9, #24 + ldrb r5, [r0, r5, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #12 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r9, #24 + lsr r6, r6, #24 #else - ubfx r4, r12, #16, #4 + uxtb r6, r9 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 +#else + ubfx r6, r9, #0, #8 +#endif + eor r5, r5, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #8 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r8, #16 + lsr r7, r7, #24 #else - ubfx r4, r12, #20, #4 + uxtb r7, r8, ror #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 +#else + ubfx r7, r8, #8, #8 +#endif + eor r5, r5, lr, lsl #16 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r11, #8 + lsr lr, lr, #24 +#else + uxtb lr, r11, ror #16 +#endif +#else + ubfx lr, r11, #16, #8 +#endif + eor r5, r5, r2, lsl #24 + lsr r2, r10, #24 + ldrb r6, [r0, r6, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] + lsr r11, r11, #24 + eor r6, r6, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #20 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r10, #24 + lsr r7, r7, #24 #else - ubfx r4, r12, #8, #4 + uxtb r7, r10 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #16 - lsr r4, r4, #28 #else - ubfx r4, r12, #12, #4 + ubfx r7, r10, #0, #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 - and r4, r12, #15 - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 - lsr r6, r10, #4 - and r3, r11, #15 - lsr r11, r11, #4 + eor r6, r6, lr, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r4, r12, #24 - lsr r4, r4, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r9, #16 + lsr lr, lr, #24 #else - ubfx r4, r12, #4, #4 + uxtb lr, r9, ror #8 #endif - eor r11, r11, r10, lsl #28 - ldr r3, [lr, r3, lsl #2] - add r4, r1, r4, lsl #4 - eor r10, r6, r9, lsl #28 - lsr r9, r9, #4 - ldm r4, {r4, r5, r6, r7} - eor r9, r9, r8, lsl #28 - eor r8, r3, r8, lsr #4 - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - # REV r8, r8 - eor r3, r8, r8, ror #16 - bic r3, r3, #0xff0000 - ror r8, r8, #8 - eor r8, r8, r3, lsr #8 - # REV r9, r9 - eor r3, r9, r9, ror #16 - bic r3, r3, #0xff0000 - ror r9, r9, #8 - eor r9, r9, r3, lsr #8 - # REV r10, r10 - eor r3, r10, r10, ror #16 - bic r3, r3, #0xff0000 - ror r10, r10, #8 - eor r10, r10, r3, lsr #8 - # REV r11, r11 - eor r3, r11, r11, ror #16 - bic r3, r3, #0xff0000 - ror r11, r11, #8 - eor r11, r11, r3, lsr #8 #else - rev r8, r8 - rev r9, r9 - rev r10, r10 - rev r11, r11 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - stm r0, {r8, r9, r10, r11} - pop {r3} - subs r3, r3, #16 - add r2, r2, #16 - bne L_GCM_gmult_len_start_block - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size GCM_gmult_len,.-GCM_gmult_len -#ifndef __APPLE__ - .text - .type L_AES_ARM32_te_gcm, %object - .size L_AES_ARM32_te_gcm, 12 + ubfx lr, r9, #8, #8 +#endif + eor r6, r6, r2, lsl #24 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r8, #8 + lsr r2, r2, #24 #else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 + uxtb r2, r8, ror #16 +#endif #else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_ARM32_te_gcm: - .long L_AES_ARM32_te_data - .text - .align 4 - .globl AES_GCM_encrypt - .type AES_GCM_encrypt, %function -AES_GCM_encrypt: - push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - ldr r12, [sp, #36] - ldr r8, [sp, #40] - mov lr, r0 - adr r0, L_AES_ARM32_te_gcm - ldr r0, [r0] - ldm r8, {r4, r5, r6, r7} + ubfx r2, r8, #16, #8 +#endif + ldrb r11, [r0, r11, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] + eor lr, lr, r11, lsl #16 + ldm r3, {r8, r9, r10, r11} + eor r7, r7, lr, lsl #8 + eor r7, r7, r2, lsl #16 + # XOR in Key Schedule + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 +#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ + pop {r1, r2, lr} + ldr r3, [sp] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r10, r4, r4, ror #16 - eor r11, r5, r5, ror #16 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - eor r4, r4, r10, lsr #8 - eor r5, r5, r11, lsr #8 + eor r8, r4, r4, ror #16 + eor r9, r5, r5, ror #16 eor r10, r6, r6, ror #16 eor r11, r7, r7, ror #16 + bic r8, r8, #0xff0000 + bic r9, r9, #0xff0000 bic r10, r10, #0xff0000 bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 ror r6, r6, #8 ror r7, r7, #8 + eor r4, r4, r8, lsr #8 + eor r5, r5, r9, lsr #8 eor r6, r6, r10, lsr #8 eor r7, r7, r11, lsr #8 #else @@ -24298,13 +16874,27 @@ AES_GCM_encrypt: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - stm r8, {r4, r5, r6, r7} - push {r3, r8} - cmp r12, #10 - beq L_AES_GCM_encrypt_start_block_128 - cmp r12, #12 - beq L_AES_GCM_encrypt_start_block_192 -L_AES_GCM_encrypt_loop_block_256: + ldr r8, [lr] + ldr r9, [lr, #4] + ldr r10, [lr, #8] + ldr r11, [lr, #12] + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + ldr r8, [sp, #4] + str r4, [r1] + str r5, [r1, #4] + str r6, [r1, #8] + str r7, [r1, #12] + ldm r8, {r4, r5, r6, r7} + subs r2, r2, #16 + add lr, lr, #16 + add r1, r1, #16 + bne L_AES_GCM_encrypt_loop_block_256 + b L_AES_GCM_encrypt_end +L_AES_GCM_encrypt_start_block_192: +L_AES_GCM_encrypt_loop_block_192: push {r1, r2, lr} ldr lr, [sp, #16] add r7, r7, #1 @@ -24315,11 +16905,11 @@ L_AES_GCM_encrypt_loop_block_256: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #6 + mov r1, #5 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_GCM_encrypt_block_nr_256: +L_AES_GCM_encrypt_block_nr_192: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -24637,7 +17227,7 @@ L_AES_GCM_encrypt_block_nr_256: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_GCM_encrypt_block_nr_256 + bne L_AES_GCM_encrypt_block_nr_192 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -24997,10 +17587,10 @@ L_AES_GCM_encrypt_block_nr_256: subs r2, r2, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_GCM_encrypt_loop_block_256 + bne L_AES_GCM_encrypt_loop_block_192 b L_AES_GCM_encrypt_end -L_AES_GCM_encrypt_start_block_192: -L_AES_GCM_encrypt_loop_block_192: +L_AES_GCM_encrypt_start_block_128: +L_AES_GCM_encrypt_loop_block_128: push {r1, r2, lr} ldr lr, [sp, #16] add r7, r7, #1 @@ -25011,11 +17601,11 @@ L_AES_GCM_encrypt_loop_block_192: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #5 + mov r1, #4 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_GCM_encrypt_block_nr_192: +L_AES_GCM_encrypt_block_nr_128: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -25333,7 +17923,7 @@ L_AES_GCM_encrypt_block_nr_192: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_GCM_encrypt_block_nr_192 + bne L_AES_GCM_encrypt_block_nr_128 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -25687,349 +18277,1054 @@ L_AES_GCM_encrypt_block_nr_192: ldr r8, [sp, #4] str r4, [r1] str r5, [r1, #4] - str r6, [r1, #8] - str r7, [r1, #12] - ldm r8, {r4, r5, r6, r7} - subs r2, r2, #16 - add lr, lr, #16 - add r1, r1, #16 - bne L_AES_GCM_encrypt_loop_block_192 - b L_AES_GCM_encrypt_end -L_AES_GCM_encrypt_start_block_128: -L_AES_GCM_encrypt_loop_block_128: - push {r1, r2, lr} - ldr lr, [sp, #16] - add r7, r7, #1 - ldm r3!, {r8, r9, r10, r11} - str r7, [lr, #12] - # Round: 0 - XOR in key schedule - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 - mov r1, #4 -#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE - bl AES_encrypt_block -#else -L_AES_GCM_encrypt_block_nr_128: -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + str r6, [r1, #8] + str r7, [r1, #12] + ldm r8, {r4, r5, r6, r7} + subs r2, r2, #16 + add lr, lr, #16 + add r1, r1, #16 + bne L_AES_GCM_encrypt_loop_block_128 +L_AES_GCM_encrypt_end: + pop {r3, r8} #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r5, #8 - lsr r8, r8, #24 -#else - uxtb r8, r5, ror #16 -#endif + eor r10, r4, r4, ror #16 + eor r11, r5, r5, ror #16 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 + eor r4, r4, r10, lsr #8 + eor r5, r5, r11, lsr #8 + eor r10, r6, r6, ror #16 + eor r11, r7, r7, ror #16 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r6, r6, #8 + ror r7, r7, #8 + eor r6, r6, r10, lsr #8 + eor r7, r7, r11, lsr #8 #else - ubfx r8, r5, #16, #8 -#endif - lsr r11, r4, #24 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r6, #16 - lsr lr, lr, #24 + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r8, {r4, r5, r6, r7} + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size AES_GCM_encrypt,.-AES_GCM_encrypt +#endif /* HAVE_AESGCM */ +#ifdef WOLFSSL_AESGCM_SIV +#ifndef __APPLE__ + .text + .type L_AES_GCMSIV_polyval_base_r, %object + .size L_AES_GCMSIV_polyval_base_r, 64 #else - uxtb lr, r6, ror #8 -#endif + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 #else - ubfx lr, r6, #8, #8 -#endif + .p2align 3 +#endif /* __APPLE__ */ +L_AES_GCMSIV_polyval_base_r: + .long 0x00000000,0x1c200000,0x38400000,0x24600000 + .long 0x70800000,0x6ca00000,0x48c00000,0x54e00000 + .long 0xe1000000,0xfd200000,0xd9400000,0xc5600000 + .long 0x91800000,0x8da00000,0xa9c00000,0xb5e00000 + .text + .align 4 + .globl AES_GCMSIV_polyval_base + .type AES_GCMSIV_polyval_base, %function +AES_GCMSIV_polyval_base: + push {r4, r5, r6, r7, r8, r9, r10, lr} + adr r6, L_AES_GCMSIV_polyval_base_r + cmp r3, #0 + beq L_AES_GCMSIV_polyval_base_done +L_AES_GCMSIV_polyval_base_loop: + ldr r10, [r2, #12] + rev r10, r10 + ldr r8, [r0] + eor r8, r8, r10 + str r8, [r0] + ldr r10, [r2, #8] + rev r10, r10 + ldr r8, [r0, #4] + eor r8, r8, r10 + str r8, [r0, #4] + ldr r10, [r2, #4] + rev r10, r10 + ldr r8, [r0, #8] + eor r8, r8, r10 + str r8, [r0, #8] + ldr r10, [r2] + rev r10, r10 + ldr r8, [r0, #12] + eor r8, r8, r10 + str r8, [r0, #12] + mov r12, #0 + mov lr, #0 + mov r4, #0 + mov r5, #0 + ldr r7, [r0, #12] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r7, #24 - lsr r2, r2, #24 -#else - uxtb r2, r7 -#endif + lsl r8, r7, #4 + lsr r8, r8, #28 #else - ubfx r2, r7, #0, #8 + ubfx r8, r7, #24, #4 #endif - ldr r8, [r0, r8, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r9, r6, #8 - lsr r9, r9, #24 -#else - uxtb r9, r6, ror #16 -#endif + lsr r8, r7, #28 #else - ubfx r9, r6, #16, #8 + ubfx r8, r7, #28, #4 #endif - eor r8, r8, r11, ror #24 - lsr r11, r5, #24 - eor r8, r8, lr, ror #8 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r7, #16 - lsr lr, lr, #24 -#else - uxtb lr, r7, ror #8 -#endif + lsl r8, r7, #12 + lsr r8, r8, #28 #else - ubfx lr, r7, #8, #8 + ubfx r8, r7, #16, #4 #endif - eor r8, r8, r2, ror #16 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r4, #24 - lsr r2, r2, #24 -#else - uxtb r2, r4 -#endif + lsl r8, r7, #8 + lsr r8, r8, #28 #else - ubfx r2, r4, #0, #8 + ubfx r8, r7, #20, #4 #endif - ldr r9, [r0, r9, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r10, r7, #8 - lsr r10, r10, #24 -#else - uxtb r10, r7, ror #16 -#endif + lsl r8, r7, #20 + lsr r8, r8, #28 #else - ubfx r10, r7, #16, #8 + ubfx r8, r7, #8, #4 #endif - eor r9, r9, r11, ror #24 - lsr r11, r6, #24 - eor r9, r9, lr, ror #8 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r4, #16 - lsr lr, lr, #24 -#else - uxtb lr, r4, ror #8 -#endif + lsl r8, r7, #16 + lsr r8, r8, #28 #else - ubfx lr, r4, #8, #8 + ubfx r8, r7, #12, #4 #endif - eor r9, r9, r2, ror #16 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r5, #24 - lsr r2, r2, #24 + lsl r8, r7, #28 + lsr r8, r8, #28 #else - uxtb r2, r5 + ubfx r8, r7, #0, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r8, r7, #24 + lsr r8, r8, #28 #else - ubfx r2, r5, #0, #8 + ubfx r8, r7, #4, #4 #endif - ldr r10, [r0, r10, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 + ldr r7, [r0, #8] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r6, #24 - lsr r6, r6, #24 + lsl r8, r7, #4 + lsr r8, r8, #28 #else - uxtb r6, r6 + ubfx r8, r7, #24, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsr r8, r7, #28 #else - ubfx r6, r6, #0, #8 + ubfx r8, r7, #28, #4 #endif - eor r10, r10, r11, ror #24 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r11, r4, #8 - lsr r11, r11, #24 + lsl r8, r7, #12 + lsr r8, r8, #28 #else - uxtb r11, r4, ror #16 + ubfx r8, r7, #16, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r8, r7, #8 + lsr r8, r8, #28 #else - ubfx r11, r4, #16, #8 + ubfx r8, r7, #20, #4 #endif - eor r10, r10, lr, ror #8 - lsr lr, r7, #24 - eor r10, r10, r2, ror #16 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r5, #16 - lsr r2, r2, #24 + lsl r8, r7, #20 + lsr r8, r8, #28 #else - uxtb r2, r5, ror #8 + ubfx r8, r7, #8, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r8, r7, #16 + lsr r8, r8, #28 #else - ubfx r2, r5, #8, #8 + ubfx r8, r7, #12, #4 #endif - ldr r6, [r0, r6, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr r2, [r0, r2, lsl #2] - eor lr, lr, r6, ror #24 - ldm r3!, {r4, r5, r6, r7} - eor r11, r11, lr, ror #24 - eor r11, r11, r2, ror #8 - # XOR in Key Schedule - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r9, #8 - lsr r4, r4, #24 + lsl r8, r7, #28 + lsr r8, r8, #28 #else - uxtb r4, r9, ror #16 + ubfx r8, r7, #0, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r8, r7, #24 + lsr r8, r8, #28 #else - ubfx r4, r9, #16, #8 + ubfx r8, r7, #4, #4 #endif - lsr r7, r8, #24 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 + ldr r7, [r0, #4] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r10, #16 - lsr lr, lr, #24 + lsl r8, r7, #4 + lsr r8, r8, #28 #else - uxtb lr, r10, ror #8 + ubfx r8, r7, #24, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsr r8, r7, #28 #else - ubfx lr, r10, #8, #8 + ubfx r8, r7, #28, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r11, #24 - lsr r2, r2, #24 + lsl r8, r7, #12 + lsr r8, r8, #28 #else - uxtb r2, r11 + ubfx r8, r7, #16, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r8, r7, #8 + lsr r8, r8, #28 #else - ubfx r2, r11, #0, #8 + ubfx r8, r7, #20, #4 #endif - ldr r4, [r0, r4, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r10, #8 - lsr r5, r5, #24 + lsl r8, r7, #20 + lsr r8, r8, #28 #else - uxtb r5, r10, ror #16 + ubfx r8, r7, #8, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r8, r7, #16 + lsr r8, r8, #28 #else - ubfx r5, r10, #16, #8 + ubfx r8, r7, #12, #4 #endif - eor r4, r4, r7, ror #24 - lsr r7, r9, #24 - eor r4, r4, lr, ror #8 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r11, #16 - lsr lr, lr, #24 + lsl r8, r7, #28 + lsr r8, r8, #28 #else - uxtb lr, r11, ror #8 + ubfx r8, r7, #0, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r8, r7, #24 + lsr r8, r8, #28 #else - ubfx lr, r11, #8, #8 + ubfx r8, r7, #4, #4 #endif - eor r4, r4, r2, ror #16 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 + ldr r7, [r0] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r8, #24 - lsr r2, r2, #24 -#else - uxtb r2, r8 -#endif + lsl r8, r7, #4 + lsr r8, r8, #28 #else - ubfx r2, r8, #0, #8 + ubfx r8, r7, #24, #4 #endif - ldr r5, [r0, r5, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r11, #8 - lsr r6, r6, #24 + lsr r8, r7, #28 #else - uxtb r6, r11, ror #16 + ubfx r8, r7, #28, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r8, r7, #12 + lsr r8, r8, #28 #else - ubfx r6, r11, #16, #8 + ubfx r8, r7, #16, #4 #endif - eor r5, r5, r7, ror #24 - lsr r7, r10, #24 - eor r5, r5, lr, ror #8 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r8, #16 - lsr lr, lr, #24 + lsl r8, r7, #8 + lsr r8, r8, #28 #else - uxtb lr, r8, ror #8 + ubfx r8, r7, #20, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r8, r7, #20 + lsr r8, r8, #28 #else - ubfx lr, r8, #8, #8 + ubfx r8, r7, #8, #4 #endif - eor r5, r5, r2, ror #16 + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r9, #24 - lsr r2, r2, #24 + lsl r8, r7, #16 + lsr r8, r8, #28 #else - uxtb r2, r9 + ubfx r8, r7, #12, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + lsl r8, r7, #28 + lsr r8, r8, #28 #else - ubfx r2, r9, #0, #8 + ubfx r8, r7, #0, #4 #endif - ldr r6, [r0, r6, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + and r8, r4, #15 + lsr r4, r4, #4 + orr r4, r4, r5, lsl #28 + lsr r5, r5, #4 + orr r5, r5, r12, lsl #28 + lsr r12, r12, #4 + orr r12, r12, lr, lsl #28 + lsr lr, lr, #4 + ldr r10, [r6, r8, lsl #2] + eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r10, r10, #24 - lsr r10, r10, #24 + lsl r8, r7, #24 + lsr r8, r8, #28 #else - uxtb r10, r10 + ubfx r8, r7, #4, #4 #endif + add r9, r1, r8, lsl #4 + ldr r10, [r9] + eor r12, r12, r10 + ldr r10, [r9, #4] + eor lr, lr, r10 + ldr r10, [r9, #8] + eor r4, r4, r10 + ldr r10, [r9, #12] + eor r5, r5, r10 + rev lr, lr + rev r12, r12 + rev r5, r5 + rev r4, r4 + str lr, [r0] + str r12, [r0, #4] + str r5, [r0, #8] + str r4, [r0, #12] + subs r3, r3, #1 + add r2, r2, #16 + bne L_AES_GCMSIV_polyval_base_loop +L_AES_GCMSIV_polyval_base_done: + pop {r4, r5, r6, r7, r8, r9, r10, pc} + .size AES_GCMSIV_polyval_base,.-AES_GCMSIV_polyval_base +#ifndef __APPLE__ + .text + .type L_AES_GCMSIV_ctr_base_te_data, %object + .size L_AES_GCMSIV_ctr_base_te_data, 1024 #else - ubfx r10, r10, #0, #8 -#endif - eor r6, r6, r7, ror #24 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r8, #8 - lsr r7, r7, #24 + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_AES_GCMSIV_ctr_base_te_data: + .long 0xa5c66363,0x84f87c7c,0x99ee7777,0x8df67b7b + .long 0x0dfff2f2,0xbdd66b6b,0xb1de6f6f,0x5491c5c5 + .long 0x50603030,0x03020101,0xa9ce6767,0x7d562b2b + .long 0x19e7fefe,0x62b5d7d7,0xe64dabab,0x9aec7676 + .long 0x458fcaca,0x9d1f8282,0x4089c9c9,0x87fa7d7d + .long 0x15effafa,0xebb25959,0xc98e4747,0x0bfbf0f0 + .long 0xec41adad,0x67b3d4d4,0xfd5fa2a2,0xea45afaf + .long 0xbf239c9c,0xf753a4a4,0x96e47272,0x5b9bc0c0 + .long 0xc275b7b7,0x1ce1fdfd,0xae3d9393,0x6a4c2626 + .long 0x5a6c3636,0x417e3f3f,0x02f5f7f7,0x4f83cccc + .long 0x5c683434,0xf451a5a5,0x34d1e5e5,0x08f9f1f1 + .long 0x93e27171,0x73abd8d8,0x53623131,0x3f2a1515 + .long 0x0c080404,0x5295c7c7,0x65462323,0x5e9dc3c3 + .long 0x28301818,0xa1379696,0x0f0a0505,0xb52f9a9a + .long 0x090e0707,0x36241212,0x9b1b8080,0x3ddfe2e2 + .long 0x26cdebeb,0x694e2727,0xcd7fb2b2,0x9fea7575 + .long 0x1b120909,0x9e1d8383,0x74582c2c,0x2e341a1a + .long 0x2d361b1b,0xb2dc6e6e,0xeeb45a5a,0xfb5ba0a0 + .long 0xf6a45252,0x4d763b3b,0x61b7d6d6,0xce7db3b3 + .long 0x7b522929,0x3edde3e3,0x715e2f2f,0x97138484 + .long 0xf5a65353,0x68b9d1d1,0x00000000,0x2cc1eded + .long 0x60402020,0x1fe3fcfc,0xc879b1b1,0xedb65b5b + .long 0xbed46a6a,0x468dcbcb,0xd967bebe,0x4b723939 + .long 0xde944a4a,0xd4984c4c,0xe8b05858,0x4a85cfcf + .long 0x6bbbd0d0,0x2ac5efef,0xe54faaaa,0x16edfbfb + .long 0xc5864343,0xd79a4d4d,0x55663333,0x94118585 + .long 0xcf8a4545,0x10e9f9f9,0x06040202,0x81fe7f7f + .long 0xf0a05050,0x44783c3c,0xba259f9f,0xe34ba8a8 + .long 0xf3a25151,0xfe5da3a3,0xc0804040,0x8a058f8f + .long 0xad3f9292,0xbc219d9d,0x48703838,0x04f1f5f5 + .long 0xdf63bcbc,0xc177b6b6,0x75afdada,0x63422121 + .long 0x30201010,0x1ae5ffff,0x0efdf3f3,0x6dbfd2d2 + .long 0x4c81cdcd,0x14180c0c,0x35261313,0x2fc3ecec + .long 0xe1be5f5f,0xa2359797,0xcc884444,0x392e1717 + .long 0x5793c4c4,0xf255a7a7,0x82fc7e7e,0x477a3d3d + .long 0xacc86464,0xe7ba5d5d,0x2b321919,0x95e67373 + .long 0xa0c06060,0x98198181,0xd19e4f4f,0x7fa3dcdc + .long 0x66442222,0x7e542a2a,0xab3b9090,0x830b8888 + .long 0xca8c4646,0x29c7eeee,0xd36bb8b8,0x3c281414 + .long 0x79a7dede,0xe2bc5e5e,0x1d160b0b,0x76addbdb + .long 0x3bdbe0e0,0x56643232,0x4e743a3a,0x1e140a0a + .long 0xdb924949,0x0a0c0606,0x6c482424,0xe4b85c5c + .long 0x5d9fc2c2,0x6ebdd3d3,0xef43acac,0xa6c46262 + .long 0xa8399191,0xa4319595,0x37d3e4e4,0x8bf27979 + .long 0x32d5e7e7,0x438bc8c8,0x596e3737,0xb7da6d6d + .long 0x8c018d8d,0x64b1d5d5,0xd29c4e4e,0xe049a9a9 + .long 0xb4d86c6c,0xfaac5656,0x07f3f4f4,0x25cfeaea + .long 0xafca6565,0x8ef47a7a,0xe947aeae,0x18100808 + .long 0xd56fbaba,0x88f07878,0x6f4a2525,0x725c2e2e + .long 0x24381c1c,0xf157a6a6,0xc773b4b4,0x5197c6c6 + .long 0x23cbe8e8,0x7ca1dddd,0x9ce87474,0x213e1f1f + .long 0xdd964b4b,0xdc61bdbd,0x860d8b8b,0x850f8a8a + .long 0x90e07070,0x427c3e3e,0xc471b5b5,0xaacc6666 + .long 0xd8904848,0x05060303,0x01f7f6f6,0x121c0e0e + .long 0xa3c26161,0x5f6a3535,0xf9ae5757,0xd069b9b9 + .long 0x91178686,0x5899c1c1,0x273a1d1d,0xb9279e9e + .long 0x38d9e1e1,0x13ebf8f8,0xb32b9898,0x33221111 + .long 0xbbd26969,0x70a9d9d9,0x89078e8e,0xa7339494 + .long 0xb62d9b9b,0x223c1e1e,0x92158787,0x20c9e9e9 + .long 0x4987cece,0xffaa5555,0x78502828,0x7aa5dfdf + .long 0x8f038c8c,0xf859a1a1,0x80098989,0x171a0d0d + .long 0xda65bfbf,0x31d7e6e6,0xc6844242,0xb8d06868 + .long 0xc3824141,0xb0299999,0x775a2d2d,0x111e0f0f + .long 0xcb7bb0b0,0xfca85454,0xd66dbbbb,0x3a2c1616 +#ifndef __APPLE__ + .text + .type L_AES_GCMSIV_ctr_base_te, %object + .size L_AES_GCMSIV_ctr_base_te, 12 #else - uxtb r7, r8, ror #16 -#endif + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 #else - ubfx r7, r8, #16, #8 -#endif - eor r6, r6, lr, ror #8 - lsr lr, r11, #24 - eor r6, r6, r2, ror #16 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + .p2align 3 +#endif /* __APPLE__ */ +L_AES_GCMSIV_ctr_base_te: + .long L_AES_GCMSIV_ctr_base_te_data + .text + .align 4 + .globl AES_GCMSIV_ctr_base + .type AES_GCMSIV_ctr_base, %function +AES_GCMSIV_ctr_base: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + ldr r12, [sp, #36] + ldr r8, [sp, #40] + mov lr, r0 + adr r0, L_AES_GCMSIV_ctr_base_te + ldr r0, [r0] + ldm r8, {r4, r5, r6, r7} #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r9, #16 - lsr r2, r2, #24 -#else - uxtb r2, r9, ror #8 -#endif + eor r10, r4, r4, ror #16 + eor r11, r5, r5, ror #16 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 + eor r4, r4, r10, lsr #8 + eor r5, r5, r11, lsr #8 + eor r10, r6, r6, ror #16 + eor r11, r7, r7, ror #16 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r6, r6, #8 + ror r7, r7, #8 + eor r6, r6, r10, lsr #8 + eor r7, r7, r11, lsr #8 #else - ubfx r2, r9, #8, #8 -#endif - ldr r10, [r0, r10, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr r2, [r0, r2, lsl #2] - eor lr, lr, r10, ror #24 + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r8, {r4, r5, r6, r7} + push {r3, r8} + cmp r12, #10 + beq L_AES_GCMSIV_ctr_base_start_block_128 + cmp r12, #12 + beq L_AES_GCMSIV_ctr_base_start_block_192 +L_AES_GCMSIV_ctr_base_loop_block_256: + push {r1, r2, lr} + ldr lr, [sp, #16] + rev r8, r4 + add r8, r8, #1 + rev r8, r8 + mov r9, r5 + mov r10, r6 + mov r11, r7 + stm lr, {r8, r9, r10, r11} ldm r3!, {r8, r9, r10, r11} - eor r7, r7, lr, ror #24 - eor r7, r7, r2, ror #8 - # XOR in Key Schedule + # Round: 0 - XOR in key schedule eor r4, r4, r8 eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - subs r1, r1, #1 - bne L_AES_GCM_encrypt_block_nr_128 + mov r1, #6 +#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE + bl AES_encrypt_block +#else +L_AES_GCMSIV_ctr_base_block_nr_256: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -26190,1212 +19485,498 @@ L_AES_GCM_encrypt_block_nr_128: eor r11, r11, r7 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r11, #24 + lsl r4, r9, #8 lsr r4, r4, #24 #else - uxtb r4, r11 -#endif -#else - ubfx r4, r11, #0, #8 -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r10, #16 - lsr r7, r7, #24 -#else - uxtb r7, r10, ror #8 -#endif -#else - ubfx r7, r10, #8, #8 -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r9, #8 - lsr lr, lr, #24 -#else - uxtb lr, r9, ror #16 -#endif -#else - ubfx lr, r9, #16, #8 -#endif - lsr r2, r8, #24 - ldrb r4, [r0, r4, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r8, #24 - lsr r5, r5, #24 -#else - uxtb r5, r8 -#endif -#else - ubfx r5, r8, #0, #8 -#endif - eor r4, r4, r7, lsl #8 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r11, #16 - lsr r7, r7, #24 -#else - uxtb r7, r11, ror #8 + uxtb r4, r9, ror #16 #endif #else - ubfx r7, r11, #8, #8 + ubfx r4, r9, #16, #8 #endif - eor r4, r4, lr, lsl #16 + lsr r7, r8, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r10, #8 + lsl lr, r10, #16 lsr lr, lr, #24 #else - uxtb lr, r10, ror #16 -#endif -#else - ubfx lr, r10, #16, #8 -#endif - eor r4, r4, r2, lsl #24 - lsr r2, r9, #24 - ldrb r5, [r0, r5, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r9, #24 - lsr r6, r6, #24 -#else - uxtb r6, r9 -#endif -#else - ubfx r6, r9, #0, #8 -#endif - eor r5, r5, r7, lsl #8 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r8, #16 - lsr r7, r7, #24 -#else - uxtb r7, r8, ror #8 + uxtb lr, r10, ror #8 #endif #else - ubfx r7, r8, #8, #8 + ubfx lr, r10, #8, #8 #endif - eor r5, r5, lr, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r11, #8 - lsr lr, lr, #24 + lsl r2, r11, #24 + lsr r2, r2, #24 #else - uxtb lr, r11, ror #16 + uxtb r2, r11 #endif #else - ubfx lr, r11, #16, #8 + ubfx r2, r11, #0, #8 #endif - eor r5, r5, r2, lsl #24 - lsr r2, r10, #24 - ldrb r6, [r0, r6, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] - lsr r11, r11, #24 - eor r6, r6, r7, lsl #8 + ldr r4, [r0, r4, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r10, #24 - lsr r7, r7, #24 + lsl r5, r10, #8 + lsr r5, r5, #24 #else - uxtb r7, r10 + uxtb r5, r10, ror #16 #endif #else - ubfx r7, r10, #0, #8 + ubfx r5, r10, #16, #8 #endif - eor r6, r6, lr, lsl #16 + eor r4, r4, r7, ror #24 + lsr r7, r9, #24 + eor r4, r4, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r9, #16 + lsl lr, r11, #16 lsr lr, lr, #24 #else - uxtb lr, r9, ror #8 + uxtb lr, r11, ror #8 #endif #else - ubfx lr, r9, #8, #8 + ubfx lr, r11, #8, #8 #endif - eor r6, r6, r2, lsl #24 + eor r4, r4, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r8, #8 + lsl r2, r8, #24 lsr r2, r2, #24 #else - uxtb r2, r8, ror #16 + uxtb r2, r8 #endif #else - ubfx r2, r8, #16, #8 + ubfx r2, r8, #0, #8 #endif - ldrb r11, [r0, r11, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] - eor lr, lr, r11, lsl #16 - ldm r3, {r8, r9, r10, r11} - eor r7, r7, lr, lsl #8 - eor r7, r7, r2, lsl #16 - # XOR in Key Schedule - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 -#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - pop {r1, r2, lr} - ldr r3, [sp] -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r8, r4, r4, ror #16 - eor r9, r5, r5, ror #16 - eor r10, r6, r6, ror #16 - eor r11, r7, r7, ror #16 - bic r8, r8, #0xff0000 - bic r9, r9, #0xff0000 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - ror r6, r6, #8 - ror r7, r7, #8 - eor r4, r4, r8, lsr #8 - eor r5, r5, r9, lsr #8 - eor r6, r6, r10, lsr #8 - eor r7, r7, r11, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - ldr r8, [lr] - ldr r9, [lr, #4] - ldr r10, [lr, #8] - ldr r11, [lr, #12] - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 - ldr r8, [sp, #4] - str r4, [r1] - str r5, [r1, #4] - str r6, [r1, #8] - str r7, [r1, #12] - ldm r8, {r4, r5, r6, r7} - subs r2, r2, #16 - add lr, lr, #16 - add r1, r1, #16 - bne L_AES_GCM_encrypt_loop_block_128 -L_AES_GCM_encrypt_end: - pop {r3, r8} + ldr r5, [r0, r5, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r10, r4, r4, ror #16 - eor r11, r5, r5, ror #16 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - eor r4, r4, r10, lsr #8 - eor r5, r5, r11, lsr #8 - eor r10, r6, r6, ror #16 - eor r11, r7, r7, ror #16 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r6, r6, #8 - ror r7, r7, #8 - eor r6, r6, r10, lsr #8 - eor r7, r7, r11, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - stm r8, {r4, r5, r6, r7} - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size AES_GCM_encrypt,.-AES_GCM_encrypt -#endif /* HAVE_AESGCM */ -#ifdef WOLFSSL_AESGCM_SIV -#ifndef __APPLE__ - .text - .type L_AES_GCMSIV_polyval_base_r, %object - .size L_AES_GCMSIV_polyval_base_r, 64 + lsl r6, r11, #8 + lsr r6, r6, #24 #else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 + uxtb r6, r11, ror #16 +#endif #else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_GCMSIV_polyval_base_r: - .long 0x00000000,0x1c200000,0x38400000,0x24600000 - .long 0x70800000,0x6ca00000,0x48c00000,0x54e00000 - .long 0xe1000000,0xfd200000,0xd9400000,0xc5600000 - .long 0x91800000,0x8da00000,0xa9c00000,0xb5e00000 - .text - .align 4 - .globl AES_GCMSIV_polyval_base - .type AES_GCMSIV_polyval_base, %function -AES_GCMSIV_polyval_base: - push {r4, r5, r6, r7, r8, r9, r10, lr} - adr r6, L_AES_GCMSIV_polyval_base_r - cmp r3, #0 - beq L_AES_GCMSIV_polyval_base_done -L_AES_GCMSIV_polyval_base_loop: - ldr r10, [r2, #12] - rev r10, r10 - ldr r8, [r0] - eor r8, r8, r10 - str r8, [r0] - ldr r10, [r2, #8] - rev r10, r10 - ldr r8, [r0, #4] - eor r8, r8, r10 - str r8, [r0, #4] - ldr r10, [r2, #4] - rev r10, r10 - ldr r8, [r0, #8] - eor r8, r8, r10 - str r8, [r0, #8] - ldr r10, [r2] - rev r10, r10 - ldr r8, [r0, #12] - eor r8, r8, r10 - str r8, [r0, #12] - mov r12, #0 - mov lr, #0 - mov r4, #0 - mov r5, #0 - ldr r7, [r0, #12] + ubfx r6, r11, #16, #8 +#endif + eor r5, r5, r7, ror #24 + lsr r7, r10, #24 + eor r5, r5, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #4 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r8, #16 + lsr lr, lr, #24 #else - ubfx r8, r7, #24, #4 + uxtb lr, r8, ror #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsr r8, r7, #28 #else - ubfx r8, r7, #28, #4 + ubfx lr, r8, #8, #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 + eor r5, r5, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #12 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r9, #24 + lsr r2, r2, #24 #else - ubfx r8, r7, #16, #4 + uxtb r2, r9 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r2, r9, #0, #8 +#endif + ldr r6, [r0, r6, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #8 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r10, r10, #24 + lsr r10, r10, #24 #else - ubfx r8, r7, #20, #4 + uxtb r10, r10 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #20 - lsr r8, r8, #28 #else - ubfx r8, r7, #8, #4 + ubfx r10, r10, #0, #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 + eor r6, r6, r7, ror #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #16 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r8, #8 + lsr r7, r7, #24 #else - ubfx r8, r7, #12, #4 + uxtb r7, r8, ror #16 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r7, r8, #16, #8 +#endif + eor r6, r6, lr, ror #8 + lsr lr, r11, #24 + eor r6, r6, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #28 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r9, #16 + lsr r2, r2, #24 #else - ubfx r8, r7, #0, #4 + uxtb r2, r9, ror #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r2, r9, #8, #8 +#endif + ldr r10, [r0, r10, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r7, [r0, r7, lsl #2] + ldr r2, [r0, r2, lsl #2] + eor lr, lr, r10, ror #24 + ldm r3!, {r8, r9, r10, r11} + eor r7, r7, lr, ror #24 + eor r7, r7, r2, ror #8 + # XOR in Key Schedule + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + subs r1, r1, #1 + bne L_AES_GCMSIV_ctr_base_block_nr_256 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #24 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r8, r5, #8 + lsr r8, r8, #24 #else - ubfx r8, r7, #4, #4 + uxtb r8, r5, ror #16 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 - ldr r7, [r0, #8] +#else + ubfx r8, r5, #16, #8 +#endif + lsr r11, r4, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #4 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r6, #16 + lsr lr, lr, #24 #else - ubfx r8, r7, #24, #4 + uxtb lr, r6, ror #8 +#endif +#else + ubfx lr, r6, #8, #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsr r8, r7, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r7, #24 + lsr r2, r2, #24 #else - ubfx r8, r7, #28, #4 + uxtb r2, r7 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r2, r7, #0, #8 +#endif + ldr r8, [r0, r8, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #12 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r9, r6, #8 + lsr r9, r9, #24 #else - ubfx r8, r7, #16, #4 + uxtb r9, r6, ror #16 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r9, r6, #16, #8 +#endif + eor r8, r8, r11, ror #24 + lsr r11, r5, #24 + eor r8, r8, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #8 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r7, #16 + lsr lr, lr, #24 #else - ubfx r8, r7, #20, #4 + uxtb lr, r7, ror #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx lr, r7, #8, #8 +#endif + eor r8, r8, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #20 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r4, #24 + lsr r2, r2, #24 #else - ubfx r8, r7, #8, #4 + uxtb r2, r4 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r2, r4, #0, #8 +#endif + ldr r9, [r0, r9, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #16 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r10, r7, #8 + lsr r10, r10, #24 #else - ubfx r8, r7, #12, #4 + uxtb r10, r7, ror #16 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r10, r7, #16, #8 +#endif + eor r9, r9, r11, ror #24 + lsr r11, r6, #24 + eor r9, r9, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #28 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r4, #16 + lsr lr, lr, #24 #else - ubfx r8, r7, #0, #4 + uxtb lr, r4, ror #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx lr, r4, #8, #8 +#endif + eor r9, r9, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #24 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r5, #24 + lsr r2, r2, #24 #else - ubfx r8, r7, #4, #4 + uxtb r2, r5 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 - ldr r7, [r0, #4] +#else + ubfx r2, r5, #0, #8 +#endif + ldr r10, [r0, r10, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #4 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r6, #24 + lsr r6, r6, #24 #else - ubfx r8, r7, #24, #4 + uxtb r6, r6 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r6, r6, #0, #8 +#endif + eor r10, r10, r11, ror #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsr r8, r7, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r11, r4, #8 + lsr r11, r11, #24 #else - ubfx r8, r7, #28, #4 + uxtb r11, r4, ror #16 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r11, r4, #16, #8 +#endif + eor r10, r10, lr, ror #8 + lsr lr, r7, #24 + eor r10, r10, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #12 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r5, #16 + lsr r2, r2, #24 #else - ubfx r8, r7, #16, #4 + uxtb r2, r5, ror #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r2, r5, #8, #8 +#endif + ldr r6, [r0, r6, lsl #2] + ldr lr, [r0, lr, lsl #2] + ldr r11, [r0, r11, lsl #2] + ldr r2, [r0, r2, lsl #2] + eor lr, lr, r6, ror #24 + ldm r3!, {r4, r5, r6, r7} + eor r11, r11, lr, ror #24 + eor r11, r11, r2, ror #8 + # XOR in Key Schedule + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #8 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r4, r11, #24 + lsr r4, r4, #24 #else - ubfx r8, r7, #20, #4 + uxtb r4, r11 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #20 - lsr r8, r8, #28 #else - ubfx r8, r7, #8, #4 + ubfx r4, r11, #0, #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #16 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r10, #16 + lsr r7, r7, #24 #else - ubfx r8, r7, #12, #4 + uxtb r7, r10, ror #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #28 - lsr r8, r8, #28 #else - ubfx r8, r7, #0, #4 + ubfx r7, r10, #8, #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #24 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r9, #8 + lsr lr, lr, #24 #else - ubfx r8, r7, #4, #4 + uxtb lr, r9, ror #16 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 - ldr r7, [r0] -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #4 - lsr r8, r8, #28 #else - ubfx r8, r7, #24, #4 + ubfx lr, r9, #16, #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 + lsr r2, r8, #24 + ldrb r4, [r0, r4, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsr r8, r7, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r5, r8, #24 + lsr r5, r5, #24 #else - ubfx r8, r7, #28, #4 + uxtb r5, r8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r5, r8, #0, #8 +#endif + eor r4, r4, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #12 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r11, #16 + lsr r7, r7, #24 #else - ubfx r8, r7, #16, #4 + uxtb r7, r11, ror #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r7, r11, #8, #8 +#endif + eor r4, r4, lr, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #8 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r10, #8 + lsr lr, lr, #24 #else - ubfx r8, r7, #20, #4 + uxtb lr, r10, ror #16 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx lr, r10, #16, #8 +#endif + eor r4, r4, r2, lsl #24 + lsr r2, r9, #24 + ldrb r5, [r0, r5, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #20 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r6, r9, #24 + lsr r6, r6, #24 #else - ubfx r8, r7, #8, #4 + uxtb r6, r9 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r6, r9, #0, #8 +#endif + eor r5, r5, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #16 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r8, #16 + lsr r7, r7, #24 #else - ubfx r8, r7, #12, #4 + uxtb r7, r8, ror #8 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx r7, r8, #8, #8 +#endif + eor r5, r5, lr, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #28 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r11, #8 + lsr lr, lr, #24 #else - ubfx r8, r7, #0, #4 + uxtb lr, r11, ror #16 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - and r8, r4, #15 - lsr r4, r4, #4 - orr r4, r4, r5, lsl #28 - lsr r5, r5, #4 - orr r5, r5, r12, lsl #28 - lsr r12, r12, #4 - orr r12, r12, lr, lsl #28 - lsr lr, lr, #4 - ldr r10, [r6, r8, lsl #2] - eor lr, lr, r10 +#else + ubfx lr, r11, #16, #8 +#endif + eor r5, r5, r2, lsl #24 + lsr r2, r10, #24 + ldrb r6, [r0, r6, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] + lsr r11, r11, #24 + eor r6, r6, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - lsl r8, r7, #24 - lsr r8, r8, #28 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r7, r10, #24 + lsr r7, r7, #24 #else - ubfx r8, r7, #4, #4 + uxtb r7, r10 #endif - add r9, r1, r8, lsl #4 - ldr r10, [r9] - eor r12, r12, r10 - ldr r10, [r9, #4] - eor lr, lr, r10 - ldr r10, [r9, #8] - eor r4, r4, r10 - ldr r10, [r9, #12] - eor r5, r5, r10 - rev lr, lr - rev r12, r12 - rev r5, r5 - rev r4, r4 - str lr, [r0] - str r12, [r0, #4] - str r5, [r0, #8] - str r4, [r0, #12] - subs r3, r3, #1 - add r2, r2, #16 - bne L_AES_GCMSIV_polyval_base_loop -L_AES_GCMSIV_polyval_base_done: - pop {r4, r5, r6, r7, r8, r9, r10, pc} - .size AES_GCMSIV_polyval_base,.-AES_GCMSIV_polyval_base -#ifndef __APPLE__ - .text - .type L_AES_GCMSIV_ctr_base_te_data, %object - .size L_AES_GCMSIV_ctr_base_te_data, 1024 #else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 + ubfx r7, r10, #0, #8 +#endif + eor r6, r6, lr, lsl #16 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl lr, r9, #16 + lsr lr, lr, #24 #else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_GCMSIV_ctr_base_te_data: - .long 0xa5c66363,0x84f87c7c,0x99ee7777,0x8df67b7b - .long 0x0dfff2f2,0xbdd66b6b,0xb1de6f6f,0x5491c5c5 - .long 0x50603030,0x03020101,0xa9ce6767,0x7d562b2b - .long 0x19e7fefe,0x62b5d7d7,0xe64dabab,0x9aec7676 - .long 0x458fcaca,0x9d1f8282,0x4089c9c9,0x87fa7d7d - .long 0x15effafa,0xebb25959,0xc98e4747,0x0bfbf0f0 - .long 0xec41adad,0x67b3d4d4,0xfd5fa2a2,0xea45afaf - .long 0xbf239c9c,0xf753a4a4,0x96e47272,0x5b9bc0c0 - .long 0xc275b7b7,0x1ce1fdfd,0xae3d9393,0x6a4c2626 - .long 0x5a6c3636,0x417e3f3f,0x02f5f7f7,0x4f83cccc - .long 0x5c683434,0xf451a5a5,0x34d1e5e5,0x08f9f1f1 - .long 0x93e27171,0x73abd8d8,0x53623131,0x3f2a1515 - .long 0x0c080404,0x5295c7c7,0x65462323,0x5e9dc3c3 - .long 0x28301818,0xa1379696,0x0f0a0505,0xb52f9a9a - .long 0x090e0707,0x36241212,0x9b1b8080,0x3ddfe2e2 - .long 0x26cdebeb,0x694e2727,0xcd7fb2b2,0x9fea7575 - .long 0x1b120909,0x9e1d8383,0x74582c2c,0x2e341a1a - .long 0x2d361b1b,0xb2dc6e6e,0xeeb45a5a,0xfb5ba0a0 - .long 0xf6a45252,0x4d763b3b,0x61b7d6d6,0xce7db3b3 - .long 0x7b522929,0x3edde3e3,0x715e2f2f,0x97138484 - .long 0xf5a65353,0x68b9d1d1,0x00000000,0x2cc1eded - .long 0x60402020,0x1fe3fcfc,0xc879b1b1,0xedb65b5b - .long 0xbed46a6a,0x468dcbcb,0xd967bebe,0x4b723939 - .long 0xde944a4a,0xd4984c4c,0xe8b05858,0x4a85cfcf - .long 0x6bbbd0d0,0x2ac5efef,0xe54faaaa,0x16edfbfb - .long 0xc5864343,0xd79a4d4d,0x55663333,0x94118585 - .long 0xcf8a4545,0x10e9f9f9,0x06040202,0x81fe7f7f - .long 0xf0a05050,0x44783c3c,0xba259f9f,0xe34ba8a8 - .long 0xf3a25151,0xfe5da3a3,0xc0804040,0x8a058f8f - .long 0xad3f9292,0xbc219d9d,0x48703838,0x04f1f5f5 - .long 0xdf63bcbc,0xc177b6b6,0x75afdada,0x63422121 - .long 0x30201010,0x1ae5ffff,0x0efdf3f3,0x6dbfd2d2 - .long 0x4c81cdcd,0x14180c0c,0x35261313,0x2fc3ecec - .long 0xe1be5f5f,0xa2359797,0xcc884444,0x392e1717 - .long 0x5793c4c4,0xf255a7a7,0x82fc7e7e,0x477a3d3d - .long 0xacc86464,0xe7ba5d5d,0x2b321919,0x95e67373 - .long 0xa0c06060,0x98198181,0xd19e4f4f,0x7fa3dcdc - .long 0x66442222,0x7e542a2a,0xab3b9090,0x830b8888 - .long 0xca8c4646,0x29c7eeee,0xd36bb8b8,0x3c281414 - .long 0x79a7dede,0xe2bc5e5e,0x1d160b0b,0x76addbdb - .long 0x3bdbe0e0,0x56643232,0x4e743a3a,0x1e140a0a - .long 0xdb924949,0x0a0c0606,0x6c482424,0xe4b85c5c - .long 0x5d9fc2c2,0x6ebdd3d3,0xef43acac,0xa6c46262 - .long 0xa8399191,0xa4319595,0x37d3e4e4,0x8bf27979 - .long 0x32d5e7e7,0x438bc8c8,0x596e3737,0xb7da6d6d - .long 0x8c018d8d,0x64b1d5d5,0xd29c4e4e,0xe049a9a9 - .long 0xb4d86c6c,0xfaac5656,0x07f3f4f4,0x25cfeaea - .long 0xafca6565,0x8ef47a7a,0xe947aeae,0x18100808 - .long 0xd56fbaba,0x88f07878,0x6f4a2525,0x725c2e2e - .long 0x24381c1c,0xf157a6a6,0xc773b4b4,0x5197c6c6 - .long 0x23cbe8e8,0x7ca1dddd,0x9ce87474,0x213e1f1f - .long 0xdd964b4b,0xdc61bdbd,0x860d8b8b,0x850f8a8a - .long 0x90e07070,0x427c3e3e,0xc471b5b5,0xaacc6666 - .long 0xd8904848,0x05060303,0x01f7f6f6,0x121c0e0e - .long 0xa3c26161,0x5f6a3535,0xf9ae5757,0xd069b9b9 - .long 0x91178686,0x5899c1c1,0x273a1d1d,0xb9279e9e - .long 0x38d9e1e1,0x13ebf8f8,0xb32b9898,0x33221111 - .long 0xbbd26969,0x70a9d9d9,0x89078e8e,0xa7339494 - .long 0xb62d9b9b,0x223c1e1e,0x92158787,0x20c9e9e9 - .long 0x4987cece,0xffaa5555,0x78502828,0x7aa5dfdf - .long 0x8f038c8c,0xf859a1a1,0x80098989,0x171a0d0d - .long 0xda65bfbf,0x31d7e6e6,0xc6844242,0xb8d06868 - .long 0xc3824141,0xb0299999,0x775a2d2d,0x111e0f0f - .long 0xcb7bb0b0,0xfca85454,0xd66dbbbb,0x3a2c1616 -#ifndef __APPLE__ - .text - .type L_AES_GCMSIV_ctr_base_te, %object - .size L_AES_GCMSIV_ctr_base_te, 12 + uxtb lr, r9, ror #8 +#endif #else - .section __DATA,__data -#endif /* __APPLE__ */ - # 8-byte aligned, 64-bit aligned -#ifndef __APPLE__ - .align 3 + ubfx lr, r9, #8, #8 +#endif + eor r6, r6, r2, lsl #24 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + lsl r2, r8, #8 + lsr r2, r2, #24 #else - .p2align 3 -#endif /* __APPLE__ */ -L_AES_GCMSIV_ctr_base_te: - .long L_AES_GCMSIV_ctr_base_te_data - .text - .align 4 - .globl AES_GCMSIV_ctr_base - .type AES_GCMSIV_ctr_base, %function -AES_GCMSIV_ctr_base: - push {r4, r5, r6, r7, r8, r9, r10, r11, lr} - ldr r12, [sp, #36] - ldr r8, [sp, #40] - mov lr, r0 - adr r0, L_AES_GCMSIV_ctr_base_te - ldr r0, [r0] - ldm r8, {r4, r5, r6, r7} + uxtb r2, r8, ror #16 +#endif +#else + ubfx r2, r8, #16, #8 +#endif + ldrb r11, [r0, r11, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] + eor lr, lr, r11, lsl #16 + ldm r3, {r8, r9, r10, r11} + eor r7, r7, lr, lsl #8 + eor r7, r7, r2, lsl #16 + # XOR in Key Schedule + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 +#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ + pop {r1, r2, lr} + ldr r3, [sp] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r10, r4, r4, ror #16 - eor r11, r5, r5, ror #16 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - eor r4, r4, r10, lsr #8 - eor r5, r5, r11, lsr #8 + eor r8, r4, r4, ror #16 + eor r9, r5, r5, ror #16 eor r10, r6, r6, ror #16 eor r11, r7, r7, ror #16 + bic r8, r8, #0xff0000 + bic r9, r9, #0xff0000 bic r10, r10, #0xff0000 bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 ror r6, r6, #8 ror r7, r7, #8 + eor r4, r4, r8, lsr #8 + eor r5, r5, r9, lsr #8 eor r6, r6, r10, lsr #8 eor r7, r7, r11, lsr #8 #else @@ -27404,13 +19985,27 @@ AES_GCMSIV_ctr_base: rev r6, r6 rev r7, r7 #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - stm r8, {r4, r5, r6, r7} - push {r3, r8} - cmp r12, #10 - beq L_AES_GCMSIV_ctr_base_start_block_128 - cmp r12, #12 - beq L_AES_GCMSIV_ctr_base_start_block_192 -L_AES_GCMSIV_ctr_base_loop_block_256: + ldr r8, [lr] + ldr r9, [lr, #4] + ldr r10, [lr, #8] + ldr r11, [lr, #12] + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + ldr r8, [sp, #4] + str r4, [r1] + str r5, [r1, #4] + str r6, [r1, #8] + str r7, [r1, #12] + ldm r8, {r4, r5, r6, r7} + subs r2, r2, #16 + add lr, lr, #16 + add r1, r1, #16 + bne L_AES_GCMSIV_ctr_base_loop_block_256 + b L_AES_GCMSIV_ctr_base_end +L_AES_GCMSIV_ctr_base_start_block_192: +L_AES_GCMSIV_ctr_base_loop_block_192: push {r1, r2, lr} ldr lr, [sp, #16] rev r8, r4 @@ -27426,11 +20021,11 @@ L_AES_GCMSIV_ctr_base_loop_block_256: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #6 + mov r1, #5 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_GCMSIV_ctr_base_block_nr_256: +L_AES_GCMSIV_ctr_base_block_nr_192: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -27748,7 +20343,7 @@ L_AES_GCMSIV_ctr_base_block_nr_256: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_GCMSIV_ctr_base_block_nr_256 + bne L_AES_GCMSIV_ctr_base_block_nr_192 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -28108,10 +20703,10 @@ L_AES_GCMSIV_ctr_base_block_nr_256: subs r2, r2, #16 add lr, lr, #16 add r1, r1, #16 - bne L_AES_GCMSIV_ctr_base_loop_block_256 + bne L_AES_GCMSIV_ctr_base_loop_block_192 b L_AES_GCMSIV_ctr_base_end -L_AES_GCMSIV_ctr_base_start_block_192: -L_AES_GCMSIV_ctr_base_loop_block_192: +L_AES_GCMSIV_ctr_base_start_block_128: +L_AES_GCMSIV_ctr_base_loop_block_128: push {r1, r2, lr} ldr lr, [sp, #16] rev r8, r4 @@ -28127,11 +20722,11 @@ L_AES_GCMSIV_ctr_base_loop_block_192: eor r5, r5, r9 eor r6, r6, r10 eor r7, r7, r11 - mov r1, #5 + mov r1, #4 #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE bl AES_encrypt_block #else -L_AES_GCMSIV_ctr_base_block_nr_192: +L_AES_GCMSIV_ctr_base_block_nr_128: #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -28449,7 +21044,7 @@ L_AES_GCMSIV_ctr_base_block_nr_192: eor r6, r6, r10 eor r7, r7, r11 subs r1, r1, #1 - bne L_AES_GCMSIV_ctr_base_block_nr_192 + bne L_AES_GCMSIV_ctr_base_block_nr_128 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) lsl r8, r5, #8 @@ -28697,848 +21292,8254 @@ L_AES_GCMSIV_ctr_base_block_nr_192: lsl r7, r8, #16 lsr r7, r7, #24 #else - uxtb r7, r8, ror #8 -#endif -#else - ubfx r7, r8, #8, #8 -#endif - eor r5, r5, lr, lsl #16 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r11, #8 - lsr lr, lr, #24 -#else - uxtb lr, r11, ror #16 -#endif -#else - ubfx lr, r11, #16, #8 -#endif - eor r5, r5, r2, lsl #24 - lsr r2, r10, #24 - ldrb r6, [r0, r6, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] - lsr r11, r11, #24 - eor r6, r6, r7, lsl #8 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r10, #24 - lsr r7, r7, #24 -#else - uxtb r7, r10 -#endif -#else - ubfx r7, r10, #0, #8 -#endif - eor r6, r6, lr, lsl #16 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r9, #16 - lsr lr, lr, #24 -#else - uxtb lr, r9, ror #8 -#endif -#else - ubfx lr, r9, #8, #8 -#endif - eor r6, r6, r2, lsl #24 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r8, #8 - lsr r2, r2, #24 -#else - uxtb r2, r8, ror #16 -#endif -#else - ubfx r2, r8, #16, #8 -#endif - ldrb r11, [r0, r11, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] - eor lr, lr, r11, lsl #16 - ldm r3, {r8, r9, r10, r11} - eor r7, r7, lr, lsl #8 - eor r7, r7, r2, lsl #16 - # XOR in Key Schedule - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 -#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - pop {r1, r2, lr} - ldr r3, [sp] -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r8, r4, r4, ror #16 - eor r9, r5, r5, ror #16 - eor r10, r6, r6, ror #16 - eor r11, r7, r7, ror #16 - bic r8, r8, #0xff0000 - bic r9, r9, #0xff0000 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - ror r6, r6, #8 - ror r7, r7, #8 - eor r4, r4, r8, lsr #8 - eor r5, r5, r9, lsr #8 - eor r6, r6, r10, lsr #8 - eor r7, r7, r11, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - ldr r8, [lr] - ldr r9, [lr, #4] - ldr r10, [lr, #8] - ldr r11, [lr, #12] - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 - ldr r8, [sp, #4] - str r4, [r1] - str r5, [r1, #4] - str r6, [r1, #8] - str r7, [r1, #12] - ldm r8, {r4, r5, r6, r7} - subs r2, r2, #16 - add lr, lr, #16 - add r1, r1, #16 - bne L_AES_GCMSIV_ctr_base_loop_block_192 - b L_AES_GCMSIV_ctr_base_end -L_AES_GCMSIV_ctr_base_start_block_128: -L_AES_GCMSIV_ctr_base_loop_block_128: - push {r1, r2, lr} - ldr lr, [sp, #16] - rev r8, r4 - add r8, r8, #1 - rev r8, r8 - mov r9, r5 - mov r10, r6 - mov r11, r7 - stm lr, {r8, r9, r10, r11} - ldm r3!, {r8, r9, r10, r11} - # Round: 0 - XOR in key schedule - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 - mov r1, #4 -#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE - bl AES_encrypt_block -#else -L_AES_GCMSIV_ctr_base_block_nr_128: -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r5, #8 - lsr r8, r8, #24 -#else - uxtb r8, r5, ror #16 -#endif -#else - ubfx r8, r5, #16, #8 -#endif - lsr r11, r4, #24 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r6, #16 - lsr lr, lr, #24 -#else - uxtb lr, r6, ror #8 -#endif -#else - ubfx lr, r6, #8, #8 -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r7, #24 - lsr r2, r2, #24 -#else - uxtb r2, r7 -#endif -#else - ubfx r2, r7, #0, #8 -#endif - ldr r8, [r0, r8, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r9, r6, #8 - lsr r9, r9, #24 -#else - uxtb r9, r6, ror #16 -#endif -#else - ubfx r9, r6, #16, #8 -#endif - eor r8, r8, r11, ror #24 - lsr r11, r5, #24 - eor r8, r8, lr, ror #8 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r7, #16 - lsr lr, lr, #24 -#else - uxtb lr, r7, ror #8 + uxtb r7, r8, ror #8 #endif #else - ubfx lr, r7, #8, #8 + ubfx r7, r8, #8, #8 #endif - eor r8, r8, r2, ror #16 + eor r5, r5, lr, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r4, #24 - lsr r2, r2, #24 + lsl lr, r11, #8 + lsr lr, lr, #24 #else - uxtb r2, r4 + uxtb lr, r11, ror #16 #endif #else - ubfx r2, r4, #0, #8 + ubfx lr, r11, #16, #8 #endif - ldr r9, [r0, r9, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] + eor r5, r5, r2, lsl #24 + lsr r2, r10, #24 + ldrb r6, [r0, r6, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] + lsr r11, r11, #24 + eor r6, r6, r7, lsl #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r10, r7, #8 - lsr r10, r10, #24 + lsl r7, r10, #24 + lsr r7, r7, #24 #else - uxtb r10, r7, ror #16 + uxtb r7, r10 #endif #else - ubfx r10, r7, #16, #8 + ubfx r7, r10, #0, #8 #endif - eor r9, r9, r11, ror #24 - lsr r11, r6, #24 - eor r9, r9, lr, ror #8 + eor r6, r6, lr, lsl #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r4, #16 + lsl lr, r9, #16 lsr lr, lr, #24 #else - uxtb lr, r4, ror #8 + uxtb lr, r9, ror #8 #endif #else - ubfx lr, r4, #8, #8 + ubfx lr, r9, #8, #8 #endif - eor r9, r9, r2, ror #16 + eor r6, r6, r2, lsl #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r5, #24 + lsl r2, r8, #8 lsr r2, r2, #24 #else - uxtb r2, r5 + uxtb r2, r8, ror #16 #endif #else - ubfx r2, r5, #0, #8 + ubfx r2, r8, #16, #8 #endif - ldr r10, [r0, r10, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + ldrb r11, [r0, r11, lsl #2] + ldrb r7, [r0, r7, lsl #2] + ldrb lr, [r0, lr, lsl #2] + ldrb r2, [r0, r2, lsl #2] + eor lr, lr, r11, lsl #16 + ldm r3, {r8, r9, r10, r11} + eor r7, r7, lr, lsl #8 + eor r7, r7, r2, lsl #16 + # XOR in Key Schedule + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 +#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ + pop {r1, r2, lr} + ldr r3, [sp] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r6, #24 - lsr r6, r6, #24 + eor r8, r4, r4, ror #16 + eor r9, r5, r5, ror #16 + eor r10, r6, r6, ror #16 + eor r11, r7, r7, ror #16 + bic r8, r8, #0xff0000 + bic r9, r9, #0xff0000 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 + ror r6, r6, #8 + ror r7, r7, #8 + eor r4, r4, r8, lsr #8 + eor r5, r5, r9, lsr #8 + eor r6, r6, r10, lsr #8 + eor r7, r7, r11, lsr #8 #else - uxtb r6, r6 + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + ldr r8, [lr] + ldr r9, [lr, #4] + ldr r10, [lr, #8] + ldr r11, [lr, #12] + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + ldr r8, [sp, #4] + str r4, [r1] + str r5, [r1, #4] + str r6, [r1, #8] + str r7, [r1, #12] + ldm r8, {r4, r5, r6, r7} + subs r2, r2, #16 + add lr, lr, #16 + add r1, r1, #16 + bne L_AES_GCMSIV_ctr_base_loop_block_128 +L_AES_GCMSIV_ctr_base_end: + pop {r3, r8} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + eor r10, r4, r4, ror #16 + eor r11, r5, r5, ror #16 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r4, r4, #8 + ror r5, r5, #8 + eor r4, r4, r10, lsr #8 + eor r5, r5, r11, lsr #8 + eor r10, r6, r6, ror #16 + eor r11, r7, r7, ror #16 + bic r10, r10, #0xff0000 + bic r11, r11, #0xff0000 + ror r6, r6, #8 + ror r7, r7, #8 + eor r6, r6, r10, lsr #8 + eor r7, r7, r11, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r8, {r4, r5, r6, r7} + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size AES_GCMSIV_ctr_base,.-AES_GCMSIV_ctr_base +#endif /* WOLFSSL_AESGCM_SIV */ +#endif /* !WOLFSSL_ARMASM_NO_BASE_IMPL || WOLFSSL_ARMASM_NO_HW_CRYPTO */ +#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO + .text + .align 4 + .globl AES_set_key_AARCH32 + .type AES_set_key_AARCH32, %function +AES_set_key_AARCH32: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + cmp r1, #24 + blt L_aes_set_key_arm32_crypto_start_128 + bgt L_aes_set_key_arm32_crypto_start_256 + ldr r4, [r0], #4 + ldr r5, [r0], #4 + ldr r6, [r0], #4 + ldr r7, [r0], #4 + ldr r8, [r0], #4 + ldr r9, [r0], #4 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r4, r5} +#else + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r6, r6, #0, #8 + strd r6, r7, [r2], #8 #endif - eor r10, r10, r11, ror #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r11, r4, #8 - lsr r11, r11, #24 + stm r2!, {r8, r9} #else - uxtb r11, r4, ror #16 + strd r8, r9, [r2], #8 #endif + vdup.32 q1, r9 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #1 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + eor r8, r8, r7 + eor r9, r9, r8 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r4, r5} #else - ubfx r11, r4, #16, #8 + strd r4, r5, [r2], #8 #endif - eor r10, r10, lr, ror #8 - lsr lr, r7, #24 - eor r10, r10, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r5, #16 - lsr r2, r2, #24 + stm r2!, {r6, r7} #else - uxtb r2, r5, ror #8 + strd r6, r7, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r8, r9} #else - ubfx r2, r5, #8, #8 + strd r8, r9, [r2], #8 #endif - ldr r6, [r0, r6, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr r2, [r0, r2, lsl #2] - eor lr, lr, r6, ror #24 - ldm r3!, {r4, r5, r6, r7} - eor r11, r11, lr, ror #24 - eor r11, r11, r2, ror #8 - # XOR in Key Schedule - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 + vdup.32 q1, r9 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #2 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + eor r8, r8, r7 + eor r9, r9, r8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r9, #8 - lsr r4, r4, #24 + stm r2!, {r4, r5} #else - uxtb r4, r9, ror #16 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r4, r9, #16, #8 + strd r6, r7, [r2], #8 #endif - lsr r7, r8, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r10, #16 - lsr lr, lr, #24 + stm r2!, {r8, r9} #else - uxtb lr, r10, ror #8 + strd r8, r9, [r2], #8 #endif + vdup.32 q1, r9 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #4 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + eor r8, r8, r7 + eor r9, r9, r8 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r4, r5} #else - ubfx lr, r10, #8, #8 + strd r4, r5, [r2], #8 #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r11, #24 - lsr r2, r2, #24 + stm r2!, {r6, r7} #else - uxtb r2, r11 + strd r6, r7, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r8, r9} #else - ubfx r2, r11, #0, #8 + strd r8, r9, [r2], #8 #endif - ldr r4, [r0, r4, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] + vdup.32 q1, r9 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #8 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + eor r8, r8, r7 + eor r9, r9, r8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r10, #8 - lsr r5, r5, #24 + stm r2!, {r4, r5} #else - uxtb r5, r10, ror #16 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r5, r10, #16, #8 + strd r6, r7, [r2], #8 #endif - eor r4, r4, r7, ror #24 - lsr r7, r9, #24 - eor r4, r4, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r11, #16 - lsr lr, lr, #24 + stm r2!, {r8, r9} #else - uxtb lr, r11, ror #8 + strd r8, r9, [r2], #8 #endif + vdup.32 q1, r9 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #16 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + eor r8, r8, r7 + eor r9, r9, r8 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r4, r5} #else - ubfx lr, r11, #8, #8 + strd r4, r5, [r2], #8 #endif - eor r4, r4, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r8, #24 - lsr r2, r2, #24 + stm r2!, {r6, r7} #else - uxtb r2, r8 + strd r6, r7, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r8, r9} #else - ubfx r2, r8, #0, #8 + strd r8, r9, [r2], #8 #endif - ldr r5, [r0, r5, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] + vdup.32 q1, r9 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #32 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + eor r8, r8, r7 + eor r9, r9, r8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r11, #8 - lsr r6, r6, #24 + stm r2!, {r4, r5} #else - uxtb r6, r11, ror #16 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r6, r11, #16, #8 + strd r6, r7, [r2], #8 #endif - eor r5, r5, r7, ror #24 - lsr r7, r10, #24 - eor r5, r5, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r8, #16 - lsr lr, lr, #24 + stm r2!, {r8, r9} #else - uxtb lr, r8, ror #8 + strd r8, r9, [r2], #8 #endif + vdup.32 q1, r9 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #0x40 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + eor r8, r8, r7 + eor r9, r9, r8 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r4, r5} #else - ubfx lr, r8, #8, #8 + strd r4, r5, [r2], #8 #endif - eor r5, r5, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r9, #24 - lsr r2, r2, #24 + stm r2!, {r6, r7} #else - uxtb r2, r9 + strd r6, r7, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r8, r9} #else - ubfx r2, r9, #0, #8 + strd r8, r9, [r2], #8 #endif - ldr r6, [r0, r6, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] + vdup.32 q1, r9 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #0x80 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r10, r10, #24 - lsr r10, r10, #24 + stm r2!, {r4, r5} #else - uxtb r10, r10 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r10, r10, #0, #8 + strd r6, r7, [r2], #8 #endif - eor r6, r6, r7, ror #24 + cmp r3, #0 + beq L_aes_set_key_arm32_crypto_done + sub r2, r2, #0xd0 + vld1.32 {q0}, [r2] + add r2, r2, #0xc0 + vld1.32 {q1}, [r2] + sub r2, r2, #0xc0 + vst1.32 {q1}, [r2] + add r2, r2, #0xc0 + vst1.32 {q0}, [r2] + sub r2, r2, #0xb0 + vld1.32 {q0}, [r2] + add r2, r2, #0xa0 + vld1.32 {q1}, [r2] + sub r2, r2, #0xa0 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0xa0 + vst1.32 {q0}, [r2] + sub r2, r2, #0x90 + vld1.32 {q0}, [r2] + add r2, r2, #0x80 + vld1.32 {q1}, [r2] + sub r2, r2, #0x80 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0x80 + vst1.32 {q0}, [r2] + sub r2, r2, #0x70 + vld1.32 {q0}, [r2] + add r2, r2, #0x60 + vld1.32 {q1}, [r2] + sub r2, r2, #0x60 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0x60 + vst1.32 {q0}, [r2] + sub r2, r2, #0x50 + vld1.32 {q0}, [r2] + add r2, r2, #0x40 + vld1.32 {q1}, [r2] + sub r2, r2, #0x40 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0x40 + vst1.32 {q0}, [r2] + sub r2, r2, #48 + vld1.32 {q0}, [r2] + add r2, r2, #32 + vld1.32 {q1}, [r2] + sub r2, r2, #32 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #32 + vst1.32 {q0}, [r2] + sub r2, r2, #16 + vld1.32 {q0}, [r2] + aesimc.8 q0, q0 + vst1.32 {q0}, [r2] + b L_aes_set_key_arm32_crypto_done +L_aes_set_key_arm32_crypto_start_256: + ldr r4, [r0], #4 + ldr r5, [r0], #4 + ldr r6, [r0], #4 + ldr r7, [r0], #4 + ldr r8, [r0], #4 + ldr r9, [r0], #4 + ldr r10, [r0], #4 + ldr r11, [r0], #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r8, #8 - lsr r7, r7, #24 + stm r2!, {r4, r5} #else - uxtb r7, r8, ror #16 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r7, r8, #16, #8 + strd r6, r7, [r2], #8 #endif - eor r6, r6, lr, ror #8 - lsr lr, r11, #24 - eor r6, r6, r2, ror #16 -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r9, #16 - lsr r2, r2, #24 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r8, r9} #else - uxtb r2, r9, ror #8 + strd r8, r9, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r10, r11} #else - ubfx r2, r9, #8, #8 + strd r10, r11, [r2], #8 #endif - ldr r10, [r0, r10, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r7, [r0, r7, lsl #2] - ldr r2, [r0, r2, lsl #2] - eor lr, lr, r10, ror #24 - ldm r3!, {r8, r9, r10, r11} - eor r7, r7, lr, ror #24 - eor r7, r7, r2, ror #8 - # XOR in Key Schedule - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 - subs r1, r1, #1 - bne L_AES_GCMSIV_ctr_base_block_nr_128 + vdup.32 q1, r11 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #1 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + eor r8, r8, r12 + eor r9, r9, r8 + eor r10, r10, r9 + eor r11, r11, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r8, r5, #8 - lsr r8, r8, #24 + stm r2!, {r4, r5} #else - uxtb r8, r5, ror #16 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r8, r5, #16, #8 + strd r6, r7, [r2], #8 #endif - lsr r11, r4, #24 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r6, #16 - lsr lr, lr, #24 + stm r2!, {r8, r9} #else - uxtb lr, r6, ror #8 + strd r8, r9, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r10, r11} #else - ubfx lr, r6, #8, #8 + strd r10, r11, [r2], #8 #endif + vdup.32 q1, r11 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #2 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + eor r8, r8, r12 + eor r9, r9, r8 + eor r10, r10, r9 + eor r11, r11, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r7, #24 - lsr r2, r2, #24 + stm r2!, {r4, r5} #else - uxtb r2, r7 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r2, r7, #0, #8 + strd r6, r7, [r2], #8 #endif - ldr r8, [r0, r8, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r9, r6, #8 - lsr r9, r9, #24 + stm r2!, {r8, r9} #else - uxtb r9, r6, ror #16 + strd r8, r9, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r10, r11} #else - ubfx r9, r6, #16, #8 + strd r10, r11, [r2], #8 #endif - eor r8, r8, r11, ror #24 - lsr r11, r5, #24 - eor r8, r8, lr, ror #8 + vdup.32 q1, r11 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #4 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + eor r8, r8, r12 + eor r9, r9, r8 + eor r10, r10, r9 + eor r11, r11, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r7, #16 - lsr lr, lr, #24 + stm r2!, {r4, r5} #else - uxtb lr, r7, ror #8 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx lr, r7, #8, #8 + strd r6, r7, [r2], #8 #endif - eor r8, r8, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r4, #24 - lsr r2, r2, #24 + stm r2!, {r8, r9} #else - uxtb r2, r4 + strd r8, r9, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r10, r11} #else - ubfx r2, r4, #0, #8 + strd r10, r11, [r2], #8 #endif - ldr r9, [r0, r9, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] + vdup.32 q1, r11 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #8 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + eor r8, r8, r12 + eor r9, r9, r8 + eor r10, r10, r9 + eor r11, r11, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r10, r7, #8 - lsr r10, r10, #24 + stm r2!, {r4, r5} #else - uxtb r10, r7, ror #16 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r10, r7, #16, #8 + strd r6, r7, [r2], #8 #endif - eor r9, r9, r11, ror #24 - lsr r11, r6, #24 - eor r9, r9, lr, ror #8 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r4, #16 - lsr lr, lr, #24 + stm r2!, {r8, r9} #else - uxtb lr, r4, ror #8 + strd r8, r9, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r10, r11} #else - ubfx lr, r4, #8, #8 + strd r10, r11, [r2], #8 #endif - eor r9, r9, r2, ror #16 + vdup.32 q1, r11 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #16 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + eor r8, r8, r12 + eor r9, r9, r8 + eor r10, r10, r9 + eor r11, r11, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r5, #24 - lsr r2, r2, #24 + stm r2!, {r4, r5} #else - uxtb r2, r5 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r2, r5, #0, #8 + strd r6, r7, [r2], #8 #endif - ldr r10, [r0, r10, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r2, [r0, r2, lsl #2] #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r6, #24 - lsr r6, r6, #24 + stm r2!, {r8, r9} #else - uxtb r6, r6 + strd r8, r9, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r10, r11} #else - ubfx r6, r6, #0, #8 + strd r10, r11, [r2], #8 #endif - eor r10, r10, r11, ror #24 + vdup.32 q1, r11 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #32 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + eor r8, r8, r12 + eor r9, r9, r8 + eor r10, r10, r9 + eor r11, r11, r10 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r11, r4, #8 - lsr r11, r11, #24 + stm r2!, {r4, r5} #else - uxtb r11, r4, ror #16 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r11, r4, #16, #8 + strd r6, r7, [r2], #8 #endif - eor r10, r10, lr, ror #8 - lsr lr, r7, #24 - eor r10, r10, r2, ror #16 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r5, #16 - lsr r2, r2, #24 + stm r2!, {r8, r9} #else - uxtb r2, r5, ror #8 + strd r8, r9, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r10, r11} #else - ubfx r2, r5, #8, #8 + strd r10, r11, [r2], #8 #endif - ldr r6, [r0, r6, lsl #2] - ldr lr, [r0, lr, lsl #2] - ldr r11, [r0, r11, lsl #2] - ldr r2, [r0, r2, lsl #2] - eor lr, lr, r6, ror #24 - ldm r3!, {r4, r5, r6, r7} - eor r11, r11, lr, ror #24 - eor r11, r11, r2, ror #8 - # XOR in Key Schedule - eor r8, r8, r4 - eor r9, r9, r5 - eor r10, r10, r6 - eor r11, r11, r7 + vdup.32 q1, r11 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #0x40 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r4, r11, #24 - lsr r4, r4, #24 + stm r2!, {r4, r5} #else - uxtb r4, r11 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r4, r11, #0, #8 + strd r6, r7, [r2], #8 #endif + cmp r3, #0 + beq L_aes_set_key_arm32_crypto_done + sub r2, r2, #0xf0 + vld1.32 {q0}, [r2] + add r2, r2, #0xe0 + vld1.32 {q1}, [r2] + sub r2, r2, #0xe0 + vst1.32 {q1}, [r2] + add r2, r2, #0xe0 + vst1.32 {q0}, [r2] + sub r2, r2, #0xd0 + vld1.32 {q0}, [r2] + add r2, r2, #0xc0 + vld1.32 {q1}, [r2] + sub r2, r2, #0xc0 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0xc0 + vst1.32 {q0}, [r2] + sub r2, r2, #0xb0 + vld1.32 {q0}, [r2] + add r2, r2, #0xa0 + vld1.32 {q1}, [r2] + sub r2, r2, #0xa0 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0xa0 + vst1.32 {q0}, [r2] + sub r2, r2, #0x90 + vld1.32 {q0}, [r2] + add r2, r2, #0x80 + vld1.32 {q1}, [r2] + sub r2, r2, #0x80 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0x80 + vst1.32 {q0}, [r2] + sub r2, r2, #0x70 + vld1.32 {q0}, [r2] + add r2, r2, #0x60 + vld1.32 {q1}, [r2] + sub r2, r2, #0x60 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0x60 + vst1.32 {q0}, [r2] + sub r2, r2, #0x50 + vld1.32 {q0}, [r2] + add r2, r2, #0x40 + vld1.32 {q1}, [r2] + sub r2, r2, #0x40 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0x40 + vst1.32 {q0}, [r2] + sub r2, r2, #48 + vld1.32 {q0}, [r2] + add r2, r2, #32 + vld1.32 {q1}, [r2] + sub r2, r2, #32 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #32 + vst1.32 {q0}, [r2] + sub r2, r2, #16 + vld1.32 {q0}, [r2] + aesimc.8 q0, q0 + vst1.32 {q0}, [r2] + b L_aes_set_key_arm32_crypto_done +L_aes_set_key_arm32_crypto_start_128: + ldr r4, [r0], #4 + ldr r5, [r0], #4 + ldr r6, [r0], #4 + ldr r7, [r0], #4 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r10, #16 - lsr r7, r7, #24 + stm r2!, {r4, r5} #else - uxtb r7, r10, ror #8 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r7, r10, #8, #8 + strd r6, r7, [r2], #8 #endif + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #1 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r9, #8 - lsr lr, lr, #24 + stm r2!, {r4, r5} #else - uxtb lr, r9, ror #16 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx lr, r9, #16, #8 + strd r6, r7, [r2], #8 #endif - lsr r2, r8, #24 - ldrb r4, [r0, r4, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #2 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r5, r8, #24 - lsr r5, r5, #24 + stm r2!, {r4, r5} #else - uxtb r5, r8 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r5, r8, #0, #8 + strd r6, r7, [r2], #8 #endif - eor r4, r4, r7, lsl #8 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #4 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r11, #16 - lsr r7, r7, #24 + stm r2!, {r4, r5} #else - uxtb r7, r11, ror #8 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r7, r11, #8, #8 + strd r6, r7, [r2], #8 #endif - eor r4, r4, lr, lsl #16 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #8 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r10, #8 - lsr lr, lr, #24 + stm r2!, {r4, r5} #else - uxtb lr, r10, ror #16 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx lr, r10, #16, #8 + strd r6, r7, [r2], #8 #endif - eor r4, r4, r2, lsl #24 - lsr r2, r9, #24 - ldrb r5, [r0, r5, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #16 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r6, r9, #24 - lsr r6, r6, #24 + stm r2!, {r4, r5} #else - uxtb r6, r9 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r6, r9, #0, #8 + strd r6, r7, [r2], #8 #endif - eor r5, r5, r7, lsl #8 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #32 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r8, #16 - lsr r7, r7, #24 + stm r2!, {r4, r5} #else - uxtb r7, r8, ror #8 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r7, r8, #8, #8 + strd r6, r7, [r2], #8 #endif - eor r5, r5, lr, lsl #16 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #0x40 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r11, #8 - lsr lr, lr, #24 + stm r2!, {r4, r5} #else - uxtb lr, r11, ror #16 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx lr, r11, #16, #8 + strd r6, r7, [r2], #8 #endif - eor r5, r5, r2, lsl #24 - lsr r2, r10, #24 - ldrb r6, [r0, r6, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] - lsr r11, r11, #24 - eor r6, r6, r7, lsl #8 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + ror r12, r12, #8 + eor r4, r4, #0x80 + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r7, r10, #24 - lsr r7, r7, #24 + stm r2!, {r4, r5} #else - uxtb r7, r10 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r7, r10, #0, #8 + strd r6, r7, [r2], #8 #endif - eor r6, r6, lr, lsl #16 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + mov lr, #27 + ror r12, r12, #8 + eor r4, r4, lr + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl lr, r9, #16 - lsr lr, lr, #24 + stm r2!, {r4, r5} #else - uxtb lr, r9, ror #8 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx lr, r9, #8, #8 + strd r6, r7, [r2], #8 #endif - eor r6, r6, r2, lsl #24 + vdup.32 q1, r7 + vmov.i32 q0, #0 + aese.8 q0, q1 + vmov.i32 r12, s0 + mov lr, #54 + ror r12, r12, #8 + eor r4, r4, lr + eor r4, r4, r12 + eor r5, r5, r4 + eor r6, r6, r5 + eor r7, r7, r6 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - lsl r2, r8, #8 - lsr r2, r2, #24 + stm r2!, {r4, r5} #else - uxtb r2, r8, ror #16 + strd r4, r5, [r2], #8 #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + stm r2!, {r6, r7} #else - ubfx r2, r8, #16, #8 + strd r6, r7, [r2], #8 #endif - ldrb r11, [r0, r11, lsl #2] - ldrb r7, [r0, r7, lsl #2] - ldrb lr, [r0, lr, lsl #2] - ldrb r2, [r0, r2, lsl #2] - eor lr, lr, r11, lsl #16 - ldm r3, {r8, r9, r10, r11} - eor r7, r7, lr, lsl #8 - eor r7, r7, r2, lsl #16 - # XOR in Key Schedule - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 -#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - pop {r1, r2, lr} - ldr r3, [sp] -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r8, r4, r4, ror #16 - eor r9, r5, r5, ror #16 - eor r10, r6, r6, ror #16 - eor r11, r7, r7, ror #16 - bic r8, r8, #0xff0000 - bic r9, r9, #0xff0000 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - ror r6, r6, #8 - ror r7, r7, #8 - eor r4, r4, r8, lsr #8 - eor r5, r5, r9, lsr #8 - eor r6, r6, r10, lsr #8 - eor r7, r7, r11, lsr #8 -#else + cmp r3, #0 + beq L_aes_set_key_arm32_crypto_done + sub r2, r2, #0xb0 + vld1.32 {q0}, [r2] + add r2, r2, #0xa0 + vld1.32 {q1}, [r2] + sub r2, r2, #0xa0 + vst1.32 {q1}, [r2] + add r2, r2, #0xa0 + vst1.32 {q0}, [r2] + sub r2, r2, #0x90 + vld1.32 {q0}, [r2] + add r2, r2, #0x80 + vld1.32 {q1}, [r2] + sub r2, r2, #0x80 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0x80 + vst1.32 {q0}, [r2] + sub r2, r2, #0x70 + vld1.32 {q0}, [r2] + add r2, r2, #0x60 + vld1.32 {q1}, [r2] + sub r2, r2, #0x60 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0x60 + vst1.32 {q0}, [r2] + sub r2, r2, #0x50 + vld1.32 {q0}, [r2] + add r2, r2, #0x40 + vld1.32 {q1}, [r2] + sub r2, r2, #0x40 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #0x40 + vst1.32 {q0}, [r2] + sub r2, r2, #48 + vld1.32 {q0}, [r2] + add r2, r2, #32 + vld1.32 {q1}, [r2] + sub r2, r2, #32 + aesimc.8 q0, q0 + aesimc.8 q1, q1 + vst1.32 {q1}, [r2] + add r2, r2, #32 + vst1.32 {q0}, [r2] + sub r2, r2, #16 + vld1.32 {q0}, [r2] + aesimc.8 q0, q0 + vst1.32 {q0}, [r2] +L_aes_set_key_arm32_crypto_done: + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size AES_set_key_AARCH32,.-AES_set_key_AARCH32 +#if defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AES_CBC) + .text + .align 4 + .globl AES_encrypt_AARCH32 + .type AES_encrypt_AARCH32, %function +AES_encrypt_AARCH32: + vpush {d8-d9} + vld1.8 {q0}, [r0] + vldm r2!, {q1-q4} + aese.8 q0, q1 + aesmc.8 q0, q0 + aese.8 q0, q2 + aesmc.8 q0, q0 + aese.8 q0, q3 + aesmc.8 q0, q0 + aese.8 q0, q4 + aesmc.8 q0, q0 + vldm r2!, {q1-q4} + aese.8 q0, q1 + aesmc.8 q0, q0 + aese.8 q0, q2 + aesmc.8 q0, q0 + aese.8 q0, q3 + aesmc.8 q0, q0 + aese.8 q0, q4 + aesmc.8 q0, q0 + subs r3, r3, #10 + vld1.32 {q1-q2}, [r2]! + aese.8 q0, q1 + aesmc.8 q0, q0 + aese.8 q0, q2 + beq L_aes_encrypt_arm32_crypto_round_done + vld1.32 {q1-q2}, [r2]! + subs r3, r3, #2 + aesmc.8 q0, q0 + aese.8 q0, q1 + aesmc.8 q0, q0 + aese.8 q0, q2 + beq L_aes_encrypt_arm32_crypto_round_done + vld1.32 {q1-q2}, [r2]! + aesmc.8 q0, q0 + aese.8 q0, q1 + aesmc.8 q0, q0 + aese.8 q0, q2 +L_aes_encrypt_arm32_crypto_round_done: + vld1.32 {q1}, [r2] + veor.32 q0, q0, q1 + vst1.8 {q0}, [r1] + vpop {d8-d9} + bx lr + .size AES_encrypt_AARCH32,.-AES_encrypt_AARCH32 +#endif /* defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AES_CBC) */ +#if !defined(WC_AES_BITSLICED) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) +#ifdef HAVE_AES_DECRYPT + .text + .align 4 + .globl AES_decrypt_AARCH32 + .type AES_decrypt_AARCH32, %function +AES_decrypt_AARCH32: + vpush {d8-d9} + vld1.8 {q0}, [r0] + vldm r2!, {q1-q4} + aesd.8 q0, q1 + aesimc.8 q0, q0 + aesd.8 q0, q2 + aesimc.8 q0, q0 + aesd.8 q0, q3 + aesimc.8 q0, q0 + aesd.8 q0, q4 + aesimc.8 q0, q0 + vldm r2!, {q1-q4} + aesd.8 q0, q1 + aesimc.8 q0, q0 + aesd.8 q0, q2 + aesimc.8 q0, q0 + aesd.8 q0, q3 + aesimc.8 q0, q0 + aesd.8 q0, q4 + aesimc.8 q0, q0 + vld1.32 {q1-q2}, [r2]! + aesd.8 q0, q1 + aesimc.8 q0, q0 + aesd.8 q0, q2 + subs r3, r3, #10 + beq L_aes_decrypt_arm32_crypto_round_done + vld1.32 {q1-q2}, [r2]! + aesimc.8 q0, q0 + aesd.8 q0, q1 + aesimc.8 q0, q0 + aesd.8 q0, q2 + subs r3, r3, #2 + beq L_aes_decrypt_arm32_crypto_round_done + vld1.32 {q1-q2}, [r2]! + aesimc.8 q0, q0 + aesd.8 q0, q1 + aesimc.8 q0, q0 + aesd.8 q0, q2 +L_aes_decrypt_arm32_crypto_round_done: + vld1.32 {q1}, [r2] + veor.32 q0, q0, q1 + vst1.8 {q0}, [r1] + vpop {d8-d9} + bx lr + .size AES_decrypt_AARCH32,.-AES_decrypt_AARCH32 +#endif /* HAVE_AES_DECRYPT */ +#endif /* !defined(WC_AES_BITSLICED) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) */ +#ifdef HAVE_AES_ECB + .text + .align 4 + .globl AES_encrypt_blocks_AARCH32 + .type AES_encrypt_blocks_AARCH32, %function +AES_encrypt_blocks_AARCH32: + vpush {d8-d15} + ldr r12, [sp, #64] + vldm.32 r3!, {q0-q7} + lsr r2, r2, #4 + cmp r12, #12 + blt L_aes_encrypt_blocks_arm32_crypto_start_128 + bgt L_aes_encrypt_blocks_arm32_crypto_start_256 + # AES_ECB_192 +#ifndef NO_AES_192 + vld1.32 {q8-q9}, [r3]! + cmp r2, #1 + beq L_aes_encrypt_blocks_arm32_crypto_192_start_1 +L_aes_encrypt_blocks_arm32_crypto_192_start_4: + cmp r2, #4 + blt L_aes_encrypt_blocks_arm32_crypto_192_start_2 + vldm.8 r0!, {q12-q15} + aese.8 q12, q0 + aesmc.8 q12, q12 + aese.8 q13, q0 + aesmc.8 q13, q13 + aese.8 q14, q0 + aesmc.8 q14, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q12, q1 + aesmc.8 q12, q12 + aese.8 q13, q1 + aesmc.8 q13, q13 + aese.8 q14, q1 + aesmc.8 q14, q14 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q12, q2 + aesmc.8 q12, q12 + aese.8 q13, q2 + aesmc.8 q13, q13 + aese.8 q14, q2 + aesmc.8 q14, q14 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q12, q3 + aesmc.8 q12, q12 + aese.8 q13, q3 + aesmc.8 q13, q13 + aese.8 q14, q3 + aesmc.8 q14, q14 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q12, q4 + aesmc.8 q12, q12 + aese.8 q13, q4 + aesmc.8 q13, q13 + aese.8 q14, q4 + aesmc.8 q14, q14 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q12, q5 + aesmc.8 q12, q12 + aese.8 q13, q5 + aesmc.8 q13, q13 + aese.8 q14, q5 + aesmc.8 q14, q14 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q12, q6 + aesmc.8 q12, q12 + aese.8 q13, q6 + aesmc.8 q13, q13 + aese.8 q14, q6 + aesmc.8 q14, q14 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q12, q7 + aesmc.8 q12, q12 + aese.8 q13, q7 + aesmc.8 q13, q13 + aese.8 q14, q7 + aesmc.8 q14, q14 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q12, q8 + aesmc.8 q12, q12 + aese.8 q13, q8 + aesmc.8 q13, q13 + aese.8 q14, q8 + aesmc.8 q14, q14 + aese.8 q15, q8 + aesmc.8 q15, q15 + vld1.32 {q10}, [r3]! + aese.8 q12, q9 + aesmc.8 q12, q12 + aese.8 q13, q9 + aesmc.8 q13, q13 + aese.8 q14, q9 + aesmc.8 q14, q14 + aese.8 q15, q9 + aesmc.8 q15, q15 + vld1.32 {q11}, [r3]! + aese.8 q12, q10 + aesmc.8 q12, q12 + aese.8 q13, q10 + aesmc.8 q13, q13 + aese.8 q14, q10 + aesmc.8 q14, q14 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.32 {q10}, [r3]! + aese.8 q12, q11 + veor.32 q12, q12, q10 + aese.8 q13, q11 + veor.32 q13, q13, q10 + aese.8 q14, q11 + veor.32 q14, q14, q10 + aese.8 q15, q11 + veor.32 q15, q15, q10 + sub r3, r3, #48 + sub r2, r2, #4 + vstm.8 r1!, {q12-q15} + cmp r2, #4 + bge L_aes_encrypt_blocks_arm32_crypto_192_start_4 +L_aes_encrypt_blocks_arm32_crypto_192_start_2: + cmp r2, #2 + blt L_aes_encrypt_blocks_arm32_crypto_192_start_1 + vld1.8 {q12-q13}, [r0]! + aese.8 q12, q0 + aesmc.8 q12, q12 + aese.8 q13, q0 + aesmc.8 q13, q13 + aese.8 q12, q1 + aesmc.8 q12, q12 + aese.8 q13, q1 + aesmc.8 q13, q13 + aese.8 q12, q2 + aesmc.8 q12, q12 + aese.8 q13, q2 + aesmc.8 q13, q13 + aese.8 q12, q3 + aesmc.8 q12, q12 + aese.8 q13, q3 + aesmc.8 q13, q13 + aese.8 q12, q4 + aesmc.8 q12, q12 + aese.8 q13, q4 + aesmc.8 q13, q13 + aese.8 q12, q5 + aesmc.8 q12, q12 + aese.8 q13, q5 + aesmc.8 q13, q13 + aese.8 q12, q6 + aesmc.8 q12, q12 + aese.8 q13, q6 + aesmc.8 q13, q13 + aese.8 q12, q7 + aesmc.8 q12, q12 + aese.8 q13, q7 + aesmc.8 q13, q13 + aese.8 q12, q8 + aesmc.8 q12, q12 + aese.8 q13, q8 + aesmc.8 q13, q13 + vld1.32 {q10}, [r3]! + aese.8 q12, q9 + aesmc.8 q12, q12 + aese.8 q13, q9 + aesmc.8 q13, q13 + vld1.32 {q11}, [r3]! + aese.8 q12, q10 + aesmc.8 q12, q12 + aese.8 q13, q10 + aesmc.8 q13, q13 + vld1.32 {q10}, [r3]! + aese.8 q12, q11 + veor.32 q12, q12, q10 + aese.8 q13, q11 + veor.32 q13, q13, q10 + sub r3, r3, #48 + sub r2, r2, #2 + vst1.8 {q12-q13}, [r1]! +L_aes_encrypt_blocks_arm32_crypto_192_start_1: + cmp r2, #0 + beq L_aes_encrypt_blocks_arm32_crypto_192_done + vld1.8 {q12}, [r0]! + aese.8 q12, q0 + aesmc.8 q12, q12 + aese.8 q12, q1 + aesmc.8 q12, q12 + aese.8 q12, q2 + aesmc.8 q12, q12 + aese.8 q12, q3 + aesmc.8 q12, q12 + aese.8 q12, q4 + aesmc.8 q12, q12 + aese.8 q12, q5 + aesmc.8 q12, q12 + aese.8 q12, q6 + aesmc.8 q12, q12 + aese.8 q12, q7 + aesmc.8 q12, q12 + aese.8 q12, q8 + aesmc.8 q12, q12 + vld1.32 {q10}, [r3]! + aese.8 q12, q9 + aesmc.8 q12, q12 + vld1.32 {q11}, [r3]! + aese.8 q12, q10 + aesmc.8 q12, q12 + vld1.32 {q10}, [r3]! + aese.8 q12, q11 + veor.32 q12, q12, q10 + sub r3, r3, #48 + vst1.8 {q12}, [r1]! +L_aes_encrypt_blocks_arm32_crypto_192_done: +#endif /* !NO_AES_192 */ + b L_aes_encrypt_blocks_arm32_crypto_done + # AES_ECB_256 +L_aes_encrypt_blocks_arm32_crypto_start_256: +#ifndef NO_AES_256 + vld1.32 {q8-q9}, [r3]! + cmp r2, #1 + beq L_aes_encrypt_blocks_arm32_crypto_256_start_1 +L_aes_encrypt_blocks_arm32_crypto_256_start_4: + cmp r2, #4 + blt L_aes_encrypt_blocks_arm32_crypto_256_start_2 + vldm.8 r0!, {q12-q15} + aese.8 q12, q0 + aesmc.8 q12, q12 + aese.8 q13, q0 + aesmc.8 q13, q13 + aese.8 q14, q0 + aesmc.8 q14, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q12, q1 + aesmc.8 q12, q12 + aese.8 q13, q1 + aesmc.8 q13, q13 + aese.8 q14, q1 + aesmc.8 q14, q14 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q12, q2 + aesmc.8 q12, q12 + aese.8 q13, q2 + aesmc.8 q13, q13 + aese.8 q14, q2 + aesmc.8 q14, q14 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q12, q3 + aesmc.8 q12, q12 + aese.8 q13, q3 + aesmc.8 q13, q13 + aese.8 q14, q3 + aesmc.8 q14, q14 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q12, q4 + aesmc.8 q12, q12 + aese.8 q13, q4 + aesmc.8 q13, q13 + aese.8 q14, q4 + aesmc.8 q14, q14 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q12, q5 + aesmc.8 q12, q12 + aese.8 q13, q5 + aesmc.8 q13, q13 + aese.8 q14, q5 + aesmc.8 q14, q14 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q12, q6 + aesmc.8 q12, q12 + aese.8 q13, q6 + aesmc.8 q13, q13 + aese.8 q14, q6 + aesmc.8 q14, q14 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q12, q7 + aesmc.8 q12, q12 + aese.8 q13, q7 + aesmc.8 q13, q13 + aese.8 q14, q7 + aesmc.8 q14, q14 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q12, q8 + aesmc.8 q12, q12 + aese.8 q13, q8 + aesmc.8 q13, q13 + aese.8 q14, q8 + aesmc.8 q14, q14 + aese.8 q15, q8 + aesmc.8 q15, q15 + vld1.32 {q10}, [r3]! + aese.8 q12, q9 + aesmc.8 q12, q12 + aese.8 q13, q9 + aesmc.8 q13, q13 + aese.8 q14, q9 + aesmc.8 q14, q14 + aese.8 q15, q9 + aesmc.8 q15, q15 + vld1.32 {q11}, [r3]! + aese.8 q12, q10 + aesmc.8 q12, q12 + aese.8 q13, q10 + aesmc.8 q13, q13 + aese.8 q14, q10 + aesmc.8 q14, q14 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.32 {q10}, [r3]! + aese.8 q12, q11 + aesmc.8 q12, q12 + aese.8 q13, q11 + aesmc.8 q13, q13 + aese.8 q14, q11 + aesmc.8 q14, q14 + aese.8 q15, q11 + aesmc.8 q15, q15 + vld1.32 {q11}, [r3]! + aese.8 q12, q10 + aesmc.8 q12, q12 + aese.8 q13, q10 + aesmc.8 q13, q13 + aese.8 q14, q10 + aesmc.8 q14, q14 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.32 {q10}, [r3]! + aese.8 q12, q11 + veor.32 q12, q12, q10 + aese.8 q13, q11 + veor.32 q13, q13, q10 + aese.8 q14, q11 + veor.32 q14, q14, q10 + aese.8 q15, q11 + veor.32 q15, q15, q10 + sub r3, r3, #0x50 + sub r2, r2, #4 + vstm.8 r1!, {q12-q15} + cmp r2, #4 + bge L_aes_encrypt_blocks_arm32_crypto_256_start_4 +L_aes_encrypt_blocks_arm32_crypto_256_start_2: + cmp r2, #2 + blt L_aes_encrypt_blocks_arm32_crypto_256_start_1 + vld1.8 {q12-q13}, [r0]! + aese.8 q12, q0 + aesmc.8 q12, q12 + aese.8 q13, q0 + aesmc.8 q13, q13 + aese.8 q12, q1 + aesmc.8 q12, q12 + aese.8 q13, q1 + aesmc.8 q13, q13 + aese.8 q12, q2 + aesmc.8 q12, q12 + aese.8 q13, q2 + aesmc.8 q13, q13 + aese.8 q12, q3 + aesmc.8 q12, q12 + aese.8 q13, q3 + aesmc.8 q13, q13 + aese.8 q12, q4 + aesmc.8 q12, q12 + aese.8 q13, q4 + aesmc.8 q13, q13 + aese.8 q12, q5 + aesmc.8 q12, q12 + aese.8 q13, q5 + aesmc.8 q13, q13 + aese.8 q12, q6 + aesmc.8 q12, q12 + aese.8 q13, q6 + aesmc.8 q13, q13 + aese.8 q12, q7 + aesmc.8 q12, q12 + aese.8 q13, q7 + aesmc.8 q13, q13 + aese.8 q12, q8 + aesmc.8 q12, q12 + aese.8 q13, q8 + aesmc.8 q13, q13 + vld1.32 {q10}, [r3]! + aese.8 q12, q9 + aesmc.8 q12, q12 + aese.8 q13, q9 + aesmc.8 q13, q13 + vld1.32 {q11}, [r3]! + aese.8 q12, q10 + aesmc.8 q12, q12 + aese.8 q13, q10 + aesmc.8 q13, q13 + vld1.32 {q10}, [r3]! + aese.8 q12, q11 + aesmc.8 q12, q12 + aese.8 q13, q11 + aesmc.8 q13, q13 + vld1.32 {q11}, [r3]! + aese.8 q12, q10 + aesmc.8 q12, q12 + aese.8 q13, q10 + aesmc.8 q13, q13 + vld1.32 {q10}, [r3]! + aese.8 q12, q11 + veor.32 q12, q12, q10 + aese.8 q13, q11 + veor.32 q13, q13, q10 + sub r3, r3, #0x50 + sub r2, r2, #2 + vst1.8 {q12-q13}, [r1]! +L_aes_encrypt_blocks_arm32_crypto_256_start_1: + cmp r2, #0 + beq L_aes_encrypt_blocks_arm32_crypto_256_done + vld1.8 {q12}, [r0]! + aese.8 q12, q0 + aesmc.8 q12, q12 + aese.8 q12, q1 + aesmc.8 q12, q12 + aese.8 q12, q2 + aesmc.8 q12, q12 + aese.8 q12, q3 + aesmc.8 q12, q12 + aese.8 q12, q4 + aesmc.8 q12, q12 + aese.8 q12, q5 + aesmc.8 q12, q12 + aese.8 q12, q6 + aesmc.8 q12, q12 + aese.8 q12, q7 + aesmc.8 q12, q12 + aese.8 q12, q8 + aesmc.8 q12, q12 + vld1.32 {q10}, [r3]! + aese.8 q12, q9 + aesmc.8 q12, q12 + vld1.32 {q11}, [r3]! + aese.8 q12, q10 + aesmc.8 q12, q12 + vld1.32 {q10}, [r3]! + aese.8 q12, q11 + aesmc.8 q12, q12 + vld1.32 {q11}, [r3]! + aese.8 q12, q10 + aesmc.8 q12, q12 + vld1.32 {q10}, [r3]! + aese.8 q12, q11 + veor.32 q12, q12, q10 + sub r3, r3, #0x50 + vst1.8 {q12}, [r1]! +L_aes_encrypt_blocks_arm32_crypto_256_done: +#endif /* !NO_AES_256 */ + b L_aes_encrypt_blocks_arm32_crypto_done + # AES_ECB_128 +L_aes_encrypt_blocks_arm32_crypto_start_128: +#ifndef NO_AES_128 + vldm.32 r3!, {q8-q10} + cmp r2, #1 + beq L_aes_encrypt_blocks_arm32_crypto_128_start_1 +L_aes_encrypt_blocks_arm32_crypto_128_start_4: + cmp r2, #4 + blt L_aes_encrypt_blocks_arm32_crypto_128_start_2 + vldm.8 r0!, {q12-q15} + aese.8 q12, q0 + aesmc.8 q12, q12 + aese.8 q13, q0 + aesmc.8 q13, q13 + aese.8 q14, q0 + aesmc.8 q14, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q12, q1 + aesmc.8 q12, q12 + aese.8 q13, q1 + aesmc.8 q13, q13 + aese.8 q14, q1 + aesmc.8 q14, q14 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q12, q2 + aesmc.8 q12, q12 + aese.8 q13, q2 + aesmc.8 q13, q13 + aese.8 q14, q2 + aesmc.8 q14, q14 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q12, q3 + aesmc.8 q12, q12 + aese.8 q13, q3 + aesmc.8 q13, q13 + aese.8 q14, q3 + aesmc.8 q14, q14 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q12, q4 + aesmc.8 q12, q12 + aese.8 q13, q4 + aesmc.8 q13, q13 + aese.8 q14, q4 + aesmc.8 q14, q14 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q12, q5 + aesmc.8 q12, q12 + aese.8 q13, q5 + aesmc.8 q13, q13 + aese.8 q14, q5 + aesmc.8 q14, q14 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q12, q6 + aesmc.8 q12, q12 + aese.8 q13, q6 + aesmc.8 q13, q13 + aese.8 q14, q6 + aesmc.8 q14, q14 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q12, q7 + aesmc.8 q12, q12 + aese.8 q13, q7 + aesmc.8 q13, q13 + aese.8 q14, q7 + aesmc.8 q14, q14 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q12, q8 + aesmc.8 q12, q12 + aese.8 q13, q8 + aesmc.8 q13, q13 + aese.8 q14, q8 + aesmc.8 q14, q14 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q12, q9 + veor.32 q12, q12, q10 + aese.8 q13, q9 + veor.32 q13, q13, q10 + aese.8 q14, q9 + veor.32 q14, q14, q10 + aese.8 q15, q9 + veor.32 q15, q15, q10 + sub r2, r2, #4 + vstm.8 r1!, {q12-q15} + cmp r2, #4 + bge L_aes_encrypt_blocks_arm32_crypto_128_start_4 +L_aes_encrypt_blocks_arm32_crypto_128_start_2: + cmp r2, #2 + blt L_aes_encrypt_blocks_arm32_crypto_128_start_1 + vld1.8 {q12-q13}, [r0]! + aese.8 q12, q0 + aesmc.8 q12, q12 + aese.8 q13, q0 + aesmc.8 q13, q13 + aese.8 q12, q1 + aesmc.8 q12, q12 + aese.8 q13, q1 + aesmc.8 q13, q13 + aese.8 q12, q2 + aesmc.8 q12, q12 + aese.8 q13, q2 + aesmc.8 q13, q13 + aese.8 q12, q3 + aesmc.8 q12, q12 + aese.8 q13, q3 + aesmc.8 q13, q13 + aese.8 q12, q4 + aesmc.8 q12, q12 + aese.8 q13, q4 + aesmc.8 q13, q13 + aese.8 q12, q5 + aesmc.8 q12, q12 + aese.8 q13, q5 + aesmc.8 q13, q13 + aese.8 q12, q6 + aesmc.8 q12, q12 + aese.8 q13, q6 + aesmc.8 q13, q13 + aese.8 q12, q7 + aesmc.8 q12, q12 + aese.8 q13, q7 + aesmc.8 q13, q13 + aese.8 q12, q8 + aesmc.8 q12, q12 + aese.8 q13, q8 + aesmc.8 q13, q13 + aese.8 q12, q9 + veor.32 q12, q12, q10 + aese.8 q13, q9 + veor.32 q13, q13, q10 + sub r2, r2, #2 + vst1.8 {q12-q13}, [r1]! +L_aes_encrypt_blocks_arm32_crypto_128_start_1: + cmp r2, #0 + beq L_aes_encrypt_blocks_arm32_crypto_128_done + vld1.8 {q12}, [r0]! + aese.8 q12, q0 + aesmc.8 q12, q12 + aese.8 q12, q1 + aesmc.8 q12, q12 + aese.8 q12, q2 + aesmc.8 q12, q12 + aese.8 q12, q3 + aesmc.8 q12, q12 + aese.8 q12, q4 + aesmc.8 q12, q12 + aese.8 q12, q5 + aesmc.8 q12, q12 + aese.8 q12, q6 + aesmc.8 q12, q12 + aese.8 q12, q7 + aesmc.8 q12, q12 + aese.8 q12, q8 + aesmc.8 q12, q12 + aese.8 q12, q9 + veor.32 q12, q12, q10 + vst1.8 {q12}, [r1]! +L_aes_encrypt_blocks_arm32_crypto_128_done: +#endif /* !NO_AES_128 */ +L_aes_encrypt_blocks_arm32_crypto_done: + vpop {d8-d15} + bx lr + .size AES_encrypt_blocks_AARCH32,.-AES_encrypt_blocks_AARCH32 +#ifdef HAVE_AES_DECRYPT + .text + .align 4 + .globl AES_decrypt_blocks_AARCH32 + .type AES_decrypt_blocks_AARCH32, %function +AES_decrypt_blocks_AARCH32: + vpush {d8-d15} + ldr r12, [sp, #64] + vldm.32 r3!, {q0-q7} + lsr r2, r2, #4 + cmp r12, #12 + blt L_aes_decrypt_blocks_arm32_crypto_start_128 + bgt L_aes_decrypt_blocks_arm32_crypto_start_256 + # AES_ECB_192 +#ifndef NO_AES_192 + vld1.32 {q8-q9}, [r3]! + cmp r2, #1 + beq L_aes_decrypt_blocks_arm32_crypto_192_start_1 + cmp r2, #4 + blt L_aes_decrypt_blocks_arm32_crypto_192_start_2 +L_aes_decrypt_blocks_arm32_crypto_192_start_4: + vldm.8 r0!, {q12-q15} + aesd.8 q12, q0 + aesimc.8 q12, q12 + aesd.8 q13, q0 + aesimc.8 q13, q13 + aesd.8 q14, q0 + aesimc.8 q14, q14 + aesd.8 q15, q0 + aesimc.8 q15, q15 + aesd.8 q12, q1 + aesimc.8 q12, q12 + aesd.8 q13, q1 + aesimc.8 q13, q13 + aesd.8 q14, q1 + aesimc.8 q14, q14 + aesd.8 q15, q1 + aesimc.8 q15, q15 + aesd.8 q12, q2 + aesimc.8 q12, q12 + aesd.8 q13, q2 + aesimc.8 q13, q13 + aesd.8 q14, q2 + aesimc.8 q14, q14 + aesd.8 q15, q2 + aesimc.8 q15, q15 + aesd.8 q12, q3 + aesimc.8 q12, q12 + aesd.8 q13, q3 + aesimc.8 q13, q13 + aesd.8 q14, q3 + aesimc.8 q14, q14 + aesd.8 q15, q3 + aesimc.8 q15, q15 + aesd.8 q12, q4 + aesimc.8 q12, q12 + aesd.8 q13, q4 + aesimc.8 q13, q13 + aesd.8 q14, q4 + aesimc.8 q14, q14 + aesd.8 q15, q4 + aesimc.8 q15, q15 + aesd.8 q12, q5 + aesimc.8 q12, q12 + aesd.8 q13, q5 + aesimc.8 q13, q13 + aesd.8 q14, q5 + aesimc.8 q14, q14 + aesd.8 q15, q5 + aesimc.8 q15, q15 + aesd.8 q12, q6 + aesimc.8 q12, q12 + aesd.8 q13, q6 + aesimc.8 q13, q13 + aesd.8 q14, q6 + aesimc.8 q14, q14 + aesd.8 q15, q6 + aesimc.8 q15, q15 + aesd.8 q12, q7 + aesimc.8 q12, q12 + aesd.8 q13, q7 + aesimc.8 q13, q13 + aesd.8 q14, q7 + aesimc.8 q14, q14 + aesd.8 q15, q7 + aesimc.8 q15, q15 + aesd.8 q12, q8 + aesimc.8 q12, q12 + aesd.8 q13, q8 + aesimc.8 q13, q13 + aesd.8 q14, q8 + aesimc.8 q14, q14 + aesd.8 q15, q8 + aesimc.8 q15, q15 + vld1.32 {q10}, [r3]! + aesd.8 q12, q9 + aesimc.8 q12, q12 + aesd.8 q13, q9 + aesimc.8 q13, q13 + aesd.8 q14, q9 + aesimc.8 q14, q14 + aesd.8 q15, q9 + aesimc.8 q15, q15 + vld1.32 {q11}, [r3]! + aesd.8 q12, q10 + aesimc.8 q12, q12 + aesd.8 q13, q10 + aesimc.8 q13, q13 + aesd.8 q14, q10 + aesimc.8 q14, q14 + aesd.8 q15, q10 + aesimc.8 q15, q15 + vld1.32 {q10}, [r3]! + aesd.8 q12, q11 + veor.32 q12, q12, q10 + aesd.8 q13, q11 + veor.32 q13, q13, q10 + aesd.8 q14, q11 + veor.32 q14, q14, q10 + aesd.8 q15, q11 + veor.32 q15, q15, q10 + sub r3, r3, #48 + sub r2, r2, #4 + vstm.8 r1!, {q12-q15} + cmp r2, #4 + bge L_aes_decrypt_blocks_arm32_crypto_192_start_4 +L_aes_decrypt_blocks_arm32_crypto_192_start_2: + cmp r2, #2 + blt L_aes_decrypt_blocks_arm32_crypto_192_start_1 + vld1.8 {q12-q13}, [r0]! + aesd.8 q12, q0 + aesimc.8 q12, q12 + aesd.8 q13, q0 + aesimc.8 q13, q13 + aesd.8 q12, q1 + aesimc.8 q12, q12 + aesd.8 q13, q1 + aesimc.8 q13, q13 + aesd.8 q12, q2 + aesimc.8 q12, q12 + aesd.8 q13, q2 + aesimc.8 q13, q13 + aesd.8 q12, q3 + aesimc.8 q12, q12 + aesd.8 q13, q3 + aesimc.8 q13, q13 + aesd.8 q12, q4 + aesimc.8 q12, q12 + aesd.8 q13, q4 + aesimc.8 q13, q13 + aesd.8 q12, q5 + aesimc.8 q12, q12 + aesd.8 q13, q5 + aesimc.8 q13, q13 + aesd.8 q12, q6 + aesimc.8 q12, q12 + aesd.8 q13, q6 + aesimc.8 q13, q13 + aesd.8 q12, q7 + aesimc.8 q12, q12 + aesd.8 q13, q7 + aesimc.8 q13, q13 + aesd.8 q12, q8 + aesimc.8 q12, q12 + aesd.8 q13, q8 + aesimc.8 q13, q13 + vld1.32 {q10}, [r3]! + aesd.8 q12, q9 + aesimc.8 q12, q12 + aesd.8 q13, q9 + aesimc.8 q13, q13 + vld1.32 {q11}, [r3]! + aesd.8 q12, q10 + aesimc.8 q12, q12 + aesd.8 q13, q10 + aesimc.8 q13, q13 + vld1.32 {q10}, [r3]! + aesd.8 q12, q11 + veor.32 q12, q12, q10 + aesd.8 q13, q11 + veor.32 q13, q13, q10 + sub r3, r3, #48 + sub r2, r2, #2 + vst1.8 {q12-q13}, [r1]! +L_aes_decrypt_blocks_arm32_crypto_192_start_1: + cmp r2, #0 + beq L_aes_decrypt_blocks_arm32_crypto_192_done + vld1.8 {q12}, [r0]! + aesd.8 q12, q0 + aesimc.8 q12, q12 + aesd.8 q12, q1 + aesimc.8 q12, q12 + aesd.8 q12, q2 + aesimc.8 q12, q12 + aesd.8 q12, q3 + aesimc.8 q12, q12 + aesd.8 q12, q4 + aesimc.8 q12, q12 + aesd.8 q12, q5 + aesimc.8 q12, q12 + aesd.8 q12, q6 + aesimc.8 q12, q12 + aesd.8 q12, q7 + aesimc.8 q12, q12 + aesd.8 q12, q8 + aesimc.8 q12, q12 + vld1.32 {q10}, [r3]! + aesd.8 q12, q9 + aesimc.8 q12, q12 + vld1.32 {q11}, [r3]! + aesd.8 q12, q10 + aesimc.8 q12, q12 + vld1.32 {q10}, [r3]! + aesd.8 q12, q11 + veor.32 q12, q12, q10 + sub r3, r3, #48 + vst1.8 {q12}, [r1]! +L_aes_decrypt_blocks_arm32_crypto_192_done: +#endif /* !NO_AES_192 */ + b L_aes_decrypt_blocks_arm32_crypto_done + # AES_ECB_256 +L_aes_decrypt_blocks_arm32_crypto_start_256: +#ifndef NO_AES_256 + vld1.32 {q8-q9}, [r3]! + cmp r2, #1 + beq L_aes_decrypt_blocks_arm32_crypto_256_start_1 + cmp r2, #4 + blt L_aes_decrypt_blocks_arm32_crypto_256_start_2 +L_aes_decrypt_blocks_arm32_crypto_256_start_4: + vldm.8 r0!, {q12-q15} + aesd.8 q12, q0 + aesimc.8 q12, q12 + aesd.8 q13, q0 + aesimc.8 q13, q13 + aesd.8 q14, q0 + aesimc.8 q14, q14 + aesd.8 q15, q0 + aesimc.8 q15, q15 + aesd.8 q12, q1 + aesimc.8 q12, q12 + aesd.8 q13, q1 + aesimc.8 q13, q13 + aesd.8 q14, q1 + aesimc.8 q14, q14 + aesd.8 q15, q1 + aesimc.8 q15, q15 + aesd.8 q12, q2 + aesimc.8 q12, q12 + aesd.8 q13, q2 + aesimc.8 q13, q13 + aesd.8 q14, q2 + aesimc.8 q14, q14 + aesd.8 q15, q2 + aesimc.8 q15, q15 + aesd.8 q12, q3 + aesimc.8 q12, q12 + aesd.8 q13, q3 + aesimc.8 q13, q13 + aesd.8 q14, q3 + aesimc.8 q14, q14 + aesd.8 q15, q3 + aesimc.8 q15, q15 + aesd.8 q12, q4 + aesimc.8 q12, q12 + aesd.8 q13, q4 + aesimc.8 q13, q13 + aesd.8 q14, q4 + aesimc.8 q14, q14 + aesd.8 q15, q4 + aesimc.8 q15, q15 + aesd.8 q12, q5 + aesimc.8 q12, q12 + aesd.8 q13, q5 + aesimc.8 q13, q13 + aesd.8 q14, q5 + aesimc.8 q14, q14 + aesd.8 q15, q5 + aesimc.8 q15, q15 + aesd.8 q12, q6 + aesimc.8 q12, q12 + aesd.8 q13, q6 + aesimc.8 q13, q13 + aesd.8 q14, q6 + aesimc.8 q14, q14 + aesd.8 q15, q6 + aesimc.8 q15, q15 + aesd.8 q12, q7 + aesimc.8 q12, q12 + aesd.8 q13, q7 + aesimc.8 q13, q13 + aesd.8 q14, q7 + aesimc.8 q14, q14 + aesd.8 q15, q7 + aesimc.8 q15, q15 + aesd.8 q12, q8 + aesimc.8 q12, q12 + aesd.8 q13, q8 + aesimc.8 q13, q13 + aesd.8 q14, q8 + aesimc.8 q14, q14 + aesd.8 q15, q8 + aesimc.8 q15, q15 + vld1.32 {q10}, [r3]! + aesd.8 q12, q9 + aesimc.8 q12, q12 + aesd.8 q13, q9 + aesimc.8 q13, q13 + aesd.8 q14, q9 + aesimc.8 q14, q14 + aesd.8 q15, q9 + aesimc.8 q15, q15 + vld1.32 {q11}, [r3]! + aesd.8 q12, q10 + aesimc.8 q12, q12 + aesd.8 q13, q10 + aesimc.8 q13, q13 + aesd.8 q14, q10 + aesimc.8 q14, q14 + aesd.8 q15, q10 + aesimc.8 q15, q15 + vld1.32 {q10}, [r3]! + aesd.8 q12, q11 + aesimc.8 q12, q12 + aesd.8 q13, q11 + aesimc.8 q13, q13 + aesd.8 q14, q11 + aesimc.8 q14, q14 + aesd.8 q15, q11 + aesimc.8 q15, q15 + vld1.32 {q11}, [r3]! + aesd.8 q12, q10 + aesimc.8 q12, q12 + aesd.8 q13, q10 + aesimc.8 q13, q13 + aesd.8 q14, q10 + aesimc.8 q14, q14 + aesd.8 q15, q10 + aesimc.8 q15, q15 + vld1.32 {q10}, [r3]! + aesd.8 q12, q11 + veor.32 q12, q12, q10 + aesd.8 q13, q11 + veor.32 q13, q13, q10 + aesd.8 q14, q11 + veor.32 q14, q14, q10 + aesd.8 q15, q11 + veor.32 q15, q15, q10 + sub r3, r3, #0x50 + sub r2, r2, #4 + vstm.8 r1!, {q12-q15} + cmp r2, #4 + bge L_aes_decrypt_blocks_arm32_crypto_256_start_4 +L_aes_decrypt_blocks_arm32_crypto_256_start_2: + cmp r2, #2 + blt L_aes_decrypt_blocks_arm32_crypto_256_start_1 + vld1.8 {q12-q13}, [r0]! + aesd.8 q12, q0 + aesimc.8 q12, q12 + aesd.8 q13, q0 + aesimc.8 q13, q13 + aesd.8 q12, q1 + aesimc.8 q12, q12 + aesd.8 q13, q1 + aesimc.8 q13, q13 + aesd.8 q12, q2 + aesimc.8 q12, q12 + aesd.8 q13, q2 + aesimc.8 q13, q13 + aesd.8 q12, q3 + aesimc.8 q12, q12 + aesd.8 q13, q3 + aesimc.8 q13, q13 + aesd.8 q12, q4 + aesimc.8 q12, q12 + aesd.8 q13, q4 + aesimc.8 q13, q13 + aesd.8 q12, q5 + aesimc.8 q12, q12 + aesd.8 q13, q5 + aesimc.8 q13, q13 + aesd.8 q12, q6 + aesimc.8 q12, q12 + aesd.8 q13, q6 + aesimc.8 q13, q13 + aesd.8 q12, q7 + aesimc.8 q12, q12 + aesd.8 q13, q7 + aesimc.8 q13, q13 + aesd.8 q12, q8 + aesimc.8 q12, q12 + aesd.8 q13, q8 + aesimc.8 q13, q13 + vld1.32 {q10}, [r3]! + aesd.8 q12, q9 + aesimc.8 q12, q12 + aesd.8 q13, q9 + aesimc.8 q13, q13 + vld1.32 {q11}, [r3]! + aesd.8 q12, q10 + aesimc.8 q12, q12 + aesd.8 q13, q10 + aesimc.8 q13, q13 + vld1.32 {q10}, [r3]! + aesd.8 q12, q11 + aesimc.8 q12, q12 + aesd.8 q13, q11 + aesimc.8 q13, q13 + vld1.32 {q11}, [r3]! + aesd.8 q12, q10 + aesimc.8 q12, q12 + aesd.8 q13, q10 + aesimc.8 q13, q13 + vld1.32 {q10}, [r3]! + aesd.8 q12, q11 + veor.32 q12, q12, q10 + aesd.8 q13, q11 + veor.32 q13, q13, q10 + sub r3, r3, #0x50 + sub r2, r2, #2 + vst1.8 {q12-q13}, [r1]! +L_aes_decrypt_blocks_arm32_crypto_256_start_1: + cmp r2, #0 + beq L_aes_decrypt_blocks_arm32_crypto_256_done + vld1.8 {q12}, [r0]! + aesd.8 q12, q0 + aesimc.8 q12, q12 + aesd.8 q12, q1 + aesimc.8 q12, q12 + aesd.8 q12, q2 + aesimc.8 q12, q12 + aesd.8 q12, q3 + aesimc.8 q12, q12 + aesd.8 q12, q4 + aesimc.8 q12, q12 + aesd.8 q12, q5 + aesimc.8 q12, q12 + aesd.8 q12, q6 + aesimc.8 q12, q12 + aesd.8 q12, q7 + aesimc.8 q12, q12 + aesd.8 q12, q8 + aesimc.8 q12, q12 + vld1.32 {q10}, [r3]! + aesd.8 q12, q9 + aesimc.8 q12, q12 + vld1.32 {q11}, [r3]! + aesd.8 q12, q10 + aesimc.8 q12, q12 + vld1.32 {q10}, [r3]! + aesd.8 q12, q11 + aesimc.8 q12, q12 + vld1.32 {q11}, [r3]! + aesd.8 q12, q10 + aesimc.8 q12, q12 + vld1.32 {q10}, [r3]! + aesd.8 q12, q11 + veor.32 q12, q12, q10 + sub r3, r3, #0x50 + vst1.8 {q12}, [r1]! +L_aes_decrypt_blocks_arm32_crypto_256_done: +#endif /* !NO_AES_256 */ + b L_aes_decrypt_blocks_arm32_crypto_done + # AES_ECB_128 +L_aes_decrypt_blocks_arm32_crypto_start_128: +#ifndef NO_AES_128 + vldm.32 r3!, {q8-q10} + cmp r2, #1 + beq L_aes_decrypt_blocks_arm32_crypto_128_start_1 + cmp r2, #4 + blt L_aes_decrypt_blocks_arm32_crypto_128_start_2 +L_aes_decrypt_blocks_arm32_crypto_128_start_4: + vldm.8 r0!, {q12-q15} + aesd.8 q12, q0 + aesimc.8 q12, q12 + aesd.8 q13, q0 + aesimc.8 q13, q13 + aesd.8 q14, q0 + aesimc.8 q14, q14 + aesd.8 q15, q0 + aesimc.8 q15, q15 + aesd.8 q12, q1 + aesimc.8 q12, q12 + aesd.8 q13, q1 + aesimc.8 q13, q13 + aesd.8 q14, q1 + aesimc.8 q14, q14 + aesd.8 q15, q1 + aesimc.8 q15, q15 + aesd.8 q12, q2 + aesimc.8 q12, q12 + aesd.8 q13, q2 + aesimc.8 q13, q13 + aesd.8 q14, q2 + aesimc.8 q14, q14 + aesd.8 q15, q2 + aesimc.8 q15, q15 + aesd.8 q12, q3 + aesimc.8 q12, q12 + aesd.8 q13, q3 + aesimc.8 q13, q13 + aesd.8 q14, q3 + aesimc.8 q14, q14 + aesd.8 q15, q3 + aesimc.8 q15, q15 + aesd.8 q12, q4 + aesimc.8 q12, q12 + aesd.8 q13, q4 + aesimc.8 q13, q13 + aesd.8 q14, q4 + aesimc.8 q14, q14 + aesd.8 q15, q4 + aesimc.8 q15, q15 + aesd.8 q12, q5 + aesimc.8 q12, q12 + aesd.8 q13, q5 + aesimc.8 q13, q13 + aesd.8 q14, q5 + aesimc.8 q14, q14 + aesd.8 q15, q5 + aesimc.8 q15, q15 + aesd.8 q12, q6 + aesimc.8 q12, q12 + aesd.8 q13, q6 + aesimc.8 q13, q13 + aesd.8 q14, q6 + aesimc.8 q14, q14 + aesd.8 q15, q6 + aesimc.8 q15, q15 + aesd.8 q12, q7 + aesimc.8 q12, q12 + aesd.8 q13, q7 + aesimc.8 q13, q13 + aesd.8 q14, q7 + aesimc.8 q14, q14 + aesd.8 q15, q7 + aesimc.8 q15, q15 + aesd.8 q12, q8 + aesimc.8 q12, q12 + aesd.8 q13, q8 + aesimc.8 q13, q13 + aesd.8 q14, q8 + aesimc.8 q14, q14 + aesd.8 q15, q8 + aesimc.8 q15, q15 + aesd.8 q12, q9 + veor.32 q12, q12, q10 + aesd.8 q13, q9 + veor.32 q13, q13, q10 + aesd.8 q14, q9 + veor.32 q14, q14, q10 + aesd.8 q15, q9 + veor.32 q15, q15, q10 + sub r2, r2, #4 + vstm.8 r1!, {q12-q15} + cmp r2, #4 + bge L_aes_decrypt_blocks_arm32_crypto_128_start_4 +L_aes_decrypt_blocks_arm32_crypto_128_start_2: + cmp r2, #2 + blt L_aes_decrypt_blocks_arm32_crypto_128_start_1 + vld1.8 {q12-q13}, [r0]! + aesd.8 q12, q0 + aesimc.8 q12, q12 + aesd.8 q13, q0 + aesimc.8 q13, q13 + aesd.8 q12, q1 + aesimc.8 q12, q12 + aesd.8 q13, q1 + aesimc.8 q13, q13 + aesd.8 q12, q2 + aesimc.8 q12, q12 + aesd.8 q13, q2 + aesimc.8 q13, q13 + aesd.8 q12, q3 + aesimc.8 q12, q12 + aesd.8 q13, q3 + aesimc.8 q13, q13 + aesd.8 q12, q4 + aesimc.8 q12, q12 + aesd.8 q13, q4 + aesimc.8 q13, q13 + aesd.8 q12, q5 + aesimc.8 q12, q12 + aesd.8 q13, q5 + aesimc.8 q13, q13 + aesd.8 q12, q6 + aesimc.8 q12, q12 + aesd.8 q13, q6 + aesimc.8 q13, q13 + aesd.8 q12, q7 + aesimc.8 q12, q12 + aesd.8 q13, q7 + aesimc.8 q13, q13 + aesd.8 q12, q8 + aesimc.8 q12, q12 + aesd.8 q13, q8 + aesimc.8 q13, q13 + aesd.8 q12, q9 + veor.32 q12, q12, q10 + aesd.8 q13, q9 + veor.32 q13, q13, q10 + sub r2, r2, #2 + vst1.8 {q12-q13}, [r1]! +L_aes_decrypt_blocks_arm32_crypto_128_start_1: + cmp r2, #0 + beq L_aes_decrypt_blocks_arm32_crypto_128_done + vld1.8 {q12}, [r0]! + aesd.8 q12, q0 + aesimc.8 q12, q12 + aesd.8 q12, q1 + aesimc.8 q12, q12 + aesd.8 q12, q2 + aesimc.8 q12, q12 + aesd.8 q12, q3 + aesimc.8 q12, q12 + aesd.8 q12, q4 + aesimc.8 q12, q12 + aesd.8 q12, q5 + aesimc.8 q12, q12 + aesd.8 q12, q6 + aesimc.8 q12, q12 + aesd.8 q12, q7 + aesimc.8 q12, q12 + aesd.8 q12, q8 + aesimc.8 q12, q12 + aesd.8 q12, q9 + veor.32 q12, q12, q10 + vst1.8 {q12}, [r1]! +L_aes_decrypt_blocks_arm32_crypto_128_done: +#endif /* !NO_AES_128 */ +L_aes_decrypt_blocks_arm32_crypto_done: + vpop {d8-d15} + bx lr + .size AES_decrypt_blocks_AARCH32,.-AES_decrypt_blocks_AARCH32 +#endif /* HAVE_AES_DECRYPT */ +#endif /* HAVE_AES_ECB */ +#ifdef HAVE_AES_CBC + .text + .align 4 + .globl AES_CBC_encrypt_AARCH32 + .type AES_CBC_encrypt_AARCH32, %function +AES_CBC_encrypt_AARCH32: + push {lr} + vpush {d8-d15} + ldr r12, [sp, #68] + ldr lr, [sp, #72] + vldm.32 r12!, {q0-q7} + vld1.32 {q15}, [r3] + subs lr, lr, #12 + lsr r2, r2, #4 + blt L_aes_cbc_encrypt_arm32_crypto_start_128 + bgt L_aes_cbc_encrypt_arm32_crypto_start_256 + # AES_CBC_192 +#ifndef NO_AES_192 + vld1.8 {q14}, [r0]! + vldm.32 r12!, {q8-q12} + cmp r2, #1 + beq L_aes_cbc_encrypt_arm32_crypto_192_start_1 + cmp r2, #4 + blt L_aes_cbc_encrypt_arm32_crypto_192_start_2 +L_aes_cbc_encrypt_arm32_crypto_192_start_4: + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q11 + veor.32 q15, q15, q12 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q11 + veor.32 q15, q15, q12 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q11 + veor.32 q15, q15, q12 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + subs r2, r2, #4 + aese.8 q15, q10 + aesmc.8 q15, q15 + aese.8 q15, q11 + veor.32 q15, q15, q12 + beq L_aes_cbc_encrypt_arm32_crypto_192_done + vld1.8 {q14}, [r0]! + cmp r2, #4 + vst1.8 {q15}, [r1]! + bge L_aes_cbc_encrypt_arm32_crypto_192_start_4 + cmp r2, #2 + blt L_aes_cbc_encrypt_arm32_crypto_192_start_1 +L_aes_cbc_encrypt_arm32_crypto_192_start_2: + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q11 + veor.32 q15, q15, q12 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + subs r2, r2, #2 + aese.8 q15, q10 + aesmc.8 q15, q15 + aese.8 q15, q11 + veor.32 q15, q15, q12 + beq L_aes_cbc_encrypt_arm32_crypto_192_done + vld1.8 {q14}, [r0]! + vst1.8 {q15}, [r1]! +L_aes_cbc_encrypt_arm32_crypto_192_start_1: + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + aese.8 q15, q11 + veor.32 q15, q15, q12 +L_aes_cbc_encrypt_arm32_crypto_192_done: + vst1.8 {q15}, [r1]! +#endif /* !NO_AES_192 */ + b L_aes_cbc_encrypt_arm32_crypto_done + # AES_CBC_256 +L_aes_cbc_encrypt_arm32_crypto_start_256: +#ifndef NO_AES_256 + vld1.8 {q14}, [r0]! + vldm.32 r12!, {q8-q11} + add r12, r12, #16 + vld1.32 {q12-q13}, [r12] + sub r12, r12, #16 + cmp r2, #1 + beq L_aes_cbc_encrypt_arm32_crypto_256_start_1 + cmp r2, #4 + blt L_aes_cbc_encrypt_arm32_crypto_256_start_2 +L_aes_cbc_encrypt_arm32_crypto_256_start_4: + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.32 {q14}, [r12] + aese.8 q15, q11 + aesmc.8 q15, q15 + aese.8 q15, q14 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q12 + veor.32 q15, q15, q13 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.32 {q14}, [r12] + aese.8 q15, q11 + aesmc.8 q15, q15 + aese.8 q15, q14 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q12 + veor.32 q15, q15, q13 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.32 {q14}, [r12] + aese.8 q15, q11 + aesmc.8 q15, q15 + aese.8 q15, q14 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q12 + veor.32 q15, q15, q13 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.32 {q14}, [r12] + aese.8 q15, q11 + aesmc.8 q15, q15 + subs r2, r2, #4 + aese.8 q15, q14 + aesmc.8 q15, q15 + aese.8 q15, q12 + veor.32 q15, q15, q13 + beq L_aes_cbc_encrypt_arm32_crypto_256_done + vld1.8 {q14}, [r0]! + cmp r2, #4 + vst1.8 {q15}, [r1]! + bge L_aes_cbc_encrypt_arm32_crypto_256_start_4 + cmp r2, #2 + blt L_aes_cbc_encrypt_arm32_crypto_256_start_1 +L_aes_cbc_encrypt_arm32_crypto_256_start_2: + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.32 {q14}, [r12] + aese.8 q15, q11 + aesmc.8 q15, q15 + aese.8 q15, q14 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q12 + veor.32 q15, q15, q13 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.32 {q14}, [r12] + aese.8 q15, q11 + aesmc.8 q15, q15 + subs r2, r2, #2 + aese.8 q15, q14 + aesmc.8 q15, q15 + aese.8 q15, q12 + veor.32 q15, q15, q13 + beq L_aes_cbc_encrypt_arm32_crypto_256_done + vld1.8 {q14}, [r0]! + vst1.8 {q15}, [r1]! +L_aes_cbc_encrypt_arm32_crypto_256_start_1: + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + aesmc.8 q15, q15 + aese.8 q15, q10 + aesmc.8 q15, q15 + vld1.32 {q14}, [r12] + aese.8 q15, q11 + aesmc.8 q15, q15 + aese.8 q15, q14 + aesmc.8 q15, q15 + aese.8 q15, q12 + veor.32 q15, q15, q13 +L_aes_cbc_encrypt_arm32_crypto_256_done: + vst1.8 {q15}, [r1]! +#endif /* !NO_AES_256 */ + b L_aes_cbc_encrypt_arm32_crypto_done + # AES_CBC_128 +L_aes_cbc_encrypt_arm32_crypto_start_128: +#ifndef NO_AES_128 + vld1.8 {q14}, [r0]! + vldm.32 r12!, {q8-q10} + cmp r2, #1 + beq L_aes_cbc_encrypt_arm32_crypto_128_start_1 + cmp r2, #4 + blt L_aes_cbc_encrypt_arm32_crypto_128_start_2 +L_aes_cbc_encrypt_arm32_crypto_128_start_4: + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q9 + veor.32 q15, q15, q10 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q9 + veor.32 q15, q15, q10 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q9 + veor.32 q15, q15, q10 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + subs r2, r2, #4 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + veor.32 q15, q15, q10 + beq L_aes_cbc_encrypt_arm32_crypto_128_done + vld1.8 {q14}, [r0]! + cmp r2, #4 + vst1.8 {q15}, [r1]! + bge L_aes_cbc_encrypt_arm32_crypto_128_start_4 + cmp r2, #2 + blt L_aes_cbc_encrypt_arm32_crypto_128_start_1 +L_aes_cbc_encrypt_arm32_crypto_128_start_2: + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + vld1.8 {q14}, [r0]! + aese.8 q15, q9 + veor.32 q15, q15, q10 + vst1.8 {q15}, [r1]! + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + subs r2, r2, #2 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + veor.32 q15, q15, q10 + beq L_aes_cbc_encrypt_arm32_crypto_128_done + vld1.8 {q14}, [r0]! + vst1.8 {q15}, [r1]! +L_aes_cbc_encrypt_arm32_crypto_128_start_1: + veor.32 q15, q15, q14 + aese.8 q15, q0 + aesmc.8 q15, q15 + aese.8 q15, q1 + aesmc.8 q15, q15 + aese.8 q15, q2 + aesmc.8 q15, q15 + aese.8 q15, q3 + aesmc.8 q15, q15 + aese.8 q15, q4 + aesmc.8 q15, q15 + aese.8 q15, q5 + aesmc.8 q15, q15 + aese.8 q15, q6 + aesmc.8 q15, q15 + aese.8 q15, q7 + aesmc.8 q15, q15 + aese.8 q15, q8 + aesmc.8 q15, q15 + aese.8 q15, q9 + veor.32 q15, q15, q10 +L_aes_cbc_encrypt_arm32_crypto_128_done: + vst1.8 {q15}, [r1]! +#endif /* !NO_AES_128 */ +L_aes_cbc_encrypt_arm32_crypto_done: + vst1.32 {q15}, [r3] + vpop {d8-d15} + pop {pc} + .size AES_CBC_encrypt_AARCH32,.-AES_CBC_encrypt_AARCH32 +#ifdef HAVE_AES_DECRYPT + .text + .align 4 + .globl AES_CBC_decrypt_AARCH32 + .type AES_CBC_decrypt_AARCH32, %function +AES_CBC_decrypt_AARCH32: + push {lr} + vpush {d8-d15} + ldr r12, [sp, #68] + ldr lr, [sp, #72] + vldm.32 r12!, {q0-q7} + vld1.32 {q13}, [r3] + lsr r2, r2, #4 + cmp lr, #12 + blt L_aes_cbc_decrypt_blocks_arm32_crypto_start_128 + bgt L_aes_cbc_decrypt_blocks_arm32_crypto_start_256 + # AES_CBC_192 +#ifndef NO_AES_192 + vld1.32 {q8}, [r12]! + cmp r2, #1 + beq L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_1 +L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_2: + vld1.8 {q14-q15}, [r0]! + vmov q11, q13 + vmov q12, q14 + vmov q13, q15 + aesd.8 q14, q0 + aesimc.8 q14, q14 + aesd.8 q15, q0 + aesimc.8 q15, q15 + aesd.8 q14, q1 + aesimc.8 q14, q14 + aesd.8 q15, q1 + aesimc.8 q15, q15 + aesd.8 q14, q2 + aesimc.8 q14, q14 + aesd.8 q15, q2 + aesimc.8 q15, q15 + aesd.8 q14, q3 + aesimc.8 q14, q14 + aesd.8 q15, q3 + aesimc.8 q15, q15 + aesd.8 q14, q4 + aesimc.8 q14, q14 + aesd.8 q15, q4 + aesimc.8 q15, q15 + aesd.8 q14, q5 + aesimc.8 q14, q14 + aesd.8 q15, q5 + aesimc.8 q15, q15 + aesd.8 q14, q6 + aesimc.8 q14, q14 + aesd.8 q15, q6 + aesimc.8 q15, q15 + aesd.8 q14, q7 + aesimc.8 q14, q14 + aesd.8 q15, q7 + aesimc.8 q15, q15 + vld1.32 {q9}, [r12]! + aesd.8 q14, q8 + aesimc.8 q14, q14 + aesd.8 q15, q8 + aesimc.8 q15, q15 + vld1.32 {q10}, [r12]! + aesd.8 q14, q9 + aesimc.8 q14, q14 + aesd.8 q15, q9 + aesimc.8 q15, q15 + vld1.32 {q9}, [r12]! + aesd.8 q14, q10 + aesimc.8 q14, q14 + aesd.8 q15, q10 + aesimc.8 q15, q15 + vld1.32 {q10}, [r12] + aesd.8 q14, q9 + aesd.8 q15, q9 + sub r2, r2, #2 + veor.32 q14, q14, q10 + veor.32 q15, q15, q10 + cmp r2, #1 + veor.32 q14, q14, q11 + veor.32 q15, q15, q12 + vst1.8 {q14-q15}, [r1]! + sub r12, r12, #48 + blt L_aes_cbc_decrypt_blocks_arm32_crypto_192_done + bgt L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_2 +L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_1: + vld1.8 {q14}, [r0]! + vmov q11, q13 + vmov q13, q14 + aesd.8 q14, q0 + aesimc.8 q14, q14 + aesd.8 q14, q1 + aesimc.8 q14, q14 + aesd.8 q14, q2 + aesimc.8 q14, q14 + aesd.8 q14, q3 + aesimc.8 q14, q14 + aesd.8 q14, q4 + aesimc.8 q14, q14 + aesd.8 q14, q5 + aesimc.8 q14, q14 + aesd.8 q14, q6 + aesimc.8 q14, q14 + aesd.8 q14, q7 + aesimc.8 q14, q14 + vld1.32 {q9}, [r12]! + aesd.8 q14, q8 + aesimc.8 q14, q14 + vld1.32 {q10}, [r12]! + aesd.8 q14, q9 + aesimc.8 q14, q14 + vld1.32 {q9}, [r12]! + aesd.8 q14, q10 + aesimc.8 q14, q14 + vld1.32 {q10}, [r12] + aesd.8 q14, q9 + veor.32 q14, q14, q10 + veor.32 q14, q14, q11 + vst1.8 {q14}, [r1]! +L_aes_cbc_decrypt_blocks_arm32_crypto_192_done: +#endif /* !NO_AES_192 */ + b L_aes_cbc_decrypt_blocks_arm32_crypto_done + # AES_CBC_256 +L_aes_cbc_decrypt_blocks_arm32_crypto_start_256: +#ifndef NO_AES_256 + vld1.32 {q8}, [r12]! + cmp r2, #1 + beq L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_1 +L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_2: + vld1.8 {q14-q15}, [r0]! + vmov q11, q13 + vmov q12, q14 + vmov q13, q15 + aesd.8 q14, q0 + aesimc.8 q14, q14 + aesd.8 q15, q0 + aesimc.8 q15, q15 + aesd.8 q14, q1 + aesimc.8 q14, q14 + aesd.8 q15, q1 + aesimc.8 q15, q15 + aesd.8 q14, q2 + aesimc.8 q14, q14 + aesd.8 q15, q2 + aesimc.8 q15, q15 + aesd.8 q14, q3 + aesimc.8 q14, q14 + aesd.8 q15, q3 + aesimc.8 q15, q15 + aesd.8 q14, q4 + aesimc.8 q14, q14 + aesd.8 q15, q4 + aesimc.8 q15, q15 + aesd.8 q14, q5 + aesimc.8 q14, q14 + aesd.8 q15, q5 + aesimc.8 q15, q15 + aesd.8 q14, q6 + aesimc.8 q14, q14 + aesd.8 q15, q6 + aesimc.8 q15, q15 + aesd.8 q14, q7 + aesimc.8 q14, q14 + aesd.8 q15, q7 + aesimc.8 q15, q15 + vld1.32 {q9}, [r12]! + aesd.8 q14, q8 + aesimc.8 q14, q14 + aesd.8 q15, q8 + aesimc.8 q15, q15 + vld1.32 {q10}, [r12]! + aesd.8 q14, q9 + aesimc.8 q14, q14 + aesd.8 q15, q9 + aesimc.8 q15, q15 + vld1.32 {q9}, [r12]! + aesd.8 q14, q10 + aesimc.8 q14, q14 + aesd.8 q15, q10 + aesimc.8 q15, q15 + vld1.32 {q10}, [r12]! + aesd.8 q14, q9 + aesimc.8 q14, q14 + aesd.8 q15, q9 + aesimc.8 q15, q15 + vld1.32 {q9}, [r12]! + aesd.8 q14, q10 + aesimc.8 q14, q14 + aesd.8 q15, q10 + aesimc.8 q15, q15 + vld1.32 {q10}, [r12] + aesd.8 q14, q9 + aesd.8 q15, q9 + sub r2, r2, #2 + veor.32 q14, q14, q10 + veor.32 q15, q15, q10 + cmp r2, #1 + veor.32 q14, q14, q11 + veor.32 q15, q15, q12 + vst1.8 {q14-q15}, [r1]! + sub r12, r12, #0x50 + blt L_aes_cbc_decrypt_blocks_arm32_crypto_256_done + bgt L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_2 +L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_1: + vld1.8 {q14}, [r0]! + vmov q11, q13 + vmov q13, q14 + aesd.8 q14, q0 + aesimc.8 q14, q14 + aesd.8 q14, q1 + aesimc.8 q14, q14 + aesd.8 q14, q2 + aesimc.8 q14, q14 + aesd.8 q14, q3 + aesimc.8 q14, q14 + aesd.8 q14, q4 + aesimc.8 q14, q14 + aesd.8 q14, q5 + aesimc.8 q14, q14 + aesd.8 q14, q6 + aesimc.8 q14, q14 + aesd.8 q14, q7 + aesimc.8 q14, q14 + vld1.32 {q9}, [r12]! + aesd.8 q14, q8 + aesimc.8 q14, q14 + vld1.32 {q10}, [r12]! + aesd.8 q14, q9 + aesimc.8 q14, q14 + vld1.32 {q9}, [r12]! + aesd.8 q14, q10 + aesimc.8 q14, q14 + vld1.32 {q10}, [r12]! + aesd.8 q14, q9 + aesimc.8 q14, q14 + vld1.32 {q9}, [r12]! + aesd.8 q14, q10 + aesimc.8 q14, q14 + vld1.32 {q10}, [r12] + aesd.8 q14, q9 + veor.32 q14, q14, q10 + veor.32 q14, q14, q11 + vst1.8 {q14}, [r1]! +L_aes_cbc_decrypt_blocks_arm32_crypto_256_done: +#endif /* !NO_AES_256 */ + b L_aes_cbc_decrypt_blocks_arm32_crypto_done + # AES_CBC_128 +L_aes_cbc_decrypt_blocks_arm32_crypto_start_128: +#ifndef NO_AES_128 + vldm.32 r12!, {q8-q10} + cmp r2, #1 + beq L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_1 +L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_2: + vld1.8 {q14-q15}, [r0]! + vmov q11, q13 + vmov q12, q14 + vmov q13, q15 + aesd.8 q14, q0 + aesimc.8 q14, q14 + aesd.8 q15, q0 + aesimc.8 q15, q15 + aesd.8 q14, q1 + aesimc.8 q14, q14 + aesd.8 q15, q1 + aesimc.8 q15, q15 + aesd.8 q14, q2 + aesimc.8 q14, q14 + aesd.8 q15, q2 + aesimc.8 q15, q15 + aesd.8 q14, q3 + aesimc.8 q14, q14 + aesd.8 q15, q3 + aesimc.8 q15, q15 + aesd.8 q14, q4 + aesimc.8 q14, q14 + aesd.8 q15, q4 + aesimc.8 q15, q15 + aesd.8 q14, q5 + aesimc.8 q14, q14 + aesd.8 q15, q5 + aesimc.8 q15, q15 + aesd.8 q14, q6 + aesimc.8 q14, q14 + aesd.8 q15, q6 + aesimc.8 q15, q15 + aesd.8 q14, q7 + aesimc.8 q14, q14 + aesd.8 q15, q7 + aesimc.8 q15, q15 + aesd.8 q14, q8 + aesimc.8 q14, q14 + aesd.8 q15, q8 + aesimc.8 q15, q15 + aesd.8 q14, q9 + aesd.8 q15, q9 + sub r2, r2, #2 + veor.32 q14, q14, q10 + veor.32 q15, q15, q10 + cmp r2, #1 + veor.32 q14, q14, q11 + veor.32 q15, q15, q12 + vst1.8 {q14-q15}, [r1]! + blt L_aes_cbc_decrypt_blocks_arm32_crypto_128_done + bgt L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_2 +L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_1: + vld1.8 {q14}, [r0]! + vmov q11, q13 + vmov q13, q14 + aesd.8 q14, q0 + aesimc.8 q14, q14 + aesd.8 q14, q1 + aesimc.8 q14, q14 + aesd.8 q14, q2 + aesimc.8 q14, q14 + aesd.8 q14, q3 + aesimc.8 q14, q14 + aesd.8 q14, q4 + aesimc.8 q14, q14 + aesd.8 q14, q5 + aesimc.8 q14, q14 + aesd.8 q14, q6 + aesimc.8 q14, q14 + aesd.8 q14, q7 + aesimc.8 q14, q14 + aesd.8 q14, q8 + aesimc.8 q14, q14 + aesd.8 q14, q9 + veor.32 q14, q14, q10 + veor.32 q14, q14, q11 + vst1.8 {q14}, [r1]! +L_aes_cbc_decrypt_blocks_arm32_crypto_128_done: +#endif /* !NO_AES_128 */ +L_aes_cbc_decrypt_blocks_arm32_crypto_done: + vst1.32 {q13}, [r3] + vpop {d8-d15} + pop {pc} + .size AES_CBC_decrypt_AARCH32,.-AES_CBC_decrypt_AARCH32 +#endif /* HAVE_AES_DECRYPT */ +#endif /* HAVE_AES_CBC */ +#ifdef WOLFSSL_AES_COUNTER + .text + .align 4 + .globl AES_CTR_encrypt_AARCH32 + .type AES_CTR_encrypt_AARCH32, %function +AES_CTR_encrypt_AARCH32: + push {r4, r5, r6, r7, r8, lr} + vpush {d8-d15} + vld1.32 {q0}, [r3] + ldr r12, [sp, #88] + vrev32.8 q2, q0 + lsr r4, r2, #4 + vmov r5, r6, d4 + and r2, r2, #15 + vmov r7, r8, d5 + vldm.32 r12!, {q3-q10} + ldr lr, [sp, #100] + cmp lr, #12 + blt L_aes_ctr_encrypt_arm32_crypto_start_128 + bgt L_aes_ctr_encrypt_arm32_crypto_start_256 + # AES_CTR_192 +#ifndef NO_AES_192 + vldm.32 r12!, {q11-q13} + mov lr, #1 + cmp r4, #1 + blt L_aes_ctr_encrypt_arm32_crypto_192_done + beq L_aes_ctr_encrypt_arm32_crypto_192_start_1 + adds r8, r8, #1 + adcs r7, r7, #0 + adcs r6, r6, #0 + adc r5, r5, #0 + vmov d3, r7, r8 + vmov d2, r5, r6 + vrev32.8 q1, q1 +L_aes_ctr_encrypt_arm32_crypto_192_start_2: + aese.8 q0, q3 + aesmc.8 q0, q0 + aese.8 q1, q3 + aesmc.8 q1, q1 + adds r8, r8, #1 + aese.8 q0, q4 + aesmc.8 q0, q0 + aese.8 q1, q4 + aesmc.8 q1, q1 + adcs r7, r7, #0 + aese.8 q0, q5 + aesmc.8 q0, q0 + aese.8 q1, q5 + aesmc.8 q1, q1 + adcs r6, r6, #0 + aese.8 q0, q6 + aesmc.8 q0, q0 + aese.8 q1, q6 + aesmc.8 q1, q1 + adc r5, r5, #0 + aese.8 q0, q7 + aesmc.8 q0, q0 + aese.8 q1, q7 + aesmc.8 q1, q1 + aese.8 q0, q8 + aesmc.8 q0, q0 + aese.8 q1, q8 + aesmc.8 q1, q1 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q1, q9 + aesmc.8 q1, q1 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q1, q10 + aesmc.8 q1, q1 + aese.8 q0, q11 + aesmc.8 q0, q0 + aese.8 q1, q11 + aesmc.8 q1, q1 + aese.8 q0, q12 + aesmc.8 q0, q0 + aese.8 q1, q12 + aesmc.8 q1, q1 + vld1.32 {q14}, [r12]! + vmov d5, r7, r8 + aese.8 q0, q13 + aesmc.8 q0, q0 + aese.8 q1, q13 + aesmc.8 q1, q1 + vld1.32 {q15}, [r12] + vmov d4, r5, r6 + aese.8 q0, q14 + aese.8 q1, q14 + sub r4, r4, #2 + veor.32 q0, q0, q15 + veor.32 q1, q1, q15 + adds r8, r8, #1 + vld1.8 {q14-q15}, [r0]! + adcs r7, r7, #0 + sub r12, r12, #16 + veor.32 q14, q14, q0 + veor.32 q15, q15, q1 + adcs r6, r6, #0 + vrev32.8 q0, q2 + adc r5, r5, #0 + vmov d2, r5, r6 + vmov d3, r7, r8 + cmp r4, #1 + vst1.8 {q14-q15}, [r1]! + vrev32.8 q1, q1 + bgt L_aes_ctr_encrypt_arm32_crypto_192_start_2 + mov lr, #0 + blt L_aes_ctr_encrypt_arm32_crypto_192_done +L_aes_ctr_encrypt_arm32_crypto_192_start_1: + aese.8 q0, q3 + aesmc.8 q0, q0 + adds r8, r8, lr + aese.8 q0, q4 + aesmc.8 q0, q0 + adcs r7, r7, #0 + aese.8 q0, q5 + aesmc.8 q0, q0 + adcs r6, r6, #0 + aese.8 q0, q6 + aesmc.8 q0, q0 + adc r5, r5, #0 + aese.8 q0, q7 + aesmc.8 q0, q0 + vmov d5, r7, r8 + aese.8 q0, q8 + aesmc.8 q0, q0 + vmov d4, r5, r6 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q0, q11 + aesmc.8 q0, q0 + aese.8 q0, q12 + aesmc.8 q0, q0 + vld1.32 {q14}, [r12]! + aese.8 q0, q13 + aesmc.8 q0, q0 + vld1.32 {q15}, [r12] + aese.8 q0, q14 + vld1.8 {q14}, [r0]! + veor.32 q0, q0, q15 + veor.32 q14, q14, q0 + sub r12, r12, #16 + vst1.8 {q14}, [r1]! + mov lr, #1 + vrev32.8 q0, q2 +L_aes_ctr_encrypt_arm32_crypto_192_done: + cmp r2, #0 + beq L_aes_ctr_encrypt_arm32_crypto_192_partial_done + ldr r4, [sp, #96] + aese.8 q0, q3 + aesmc.8 q0, q0 + adds r8, r8, lr + aese.8 q0, q4 + aesmc.8 q0, q0 + adcs r7, r7, #0 + aese.8 q0, q5 + aesmc.8 q0, q0 + adcs r6, r6, #0 + aese.8 q0, q6 + aesmc.8 q0, q0 + adc r5, r5, #0 + aese.8 q0, q7 + aesmc.8 q0, q0 + vmov d5, r7, r8 + aese.8 q0, q8 + aesmc.8 q0, q0 + vmov d4, r5, r6 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q0, q11 + aesmc.8 q0, q0 + aese.8 q0, q12 + aesmc.8 q0, q0 + vld1.32 {q14}, [r12]! + aese.8 q0, q13 + aesmc.8 q0, q0 + vld1.32 {q15}, [r12] + ldr lr, [sp, #92] + aese.8 q0, q14 + veor.32 q0, q0, q15 + vst1.32 {q0}, [lr] + vmov q0, q2 + mov r5, #16 + sub r5, r5, r2 +L_aes_ctr_encrypt_arm32_crypto_192_start_byte: + ldrb r7, [lr], #1 + ldrb r8, [r0], #1 + eor r7, r7, r8 + subs r2, r2, #1 + strb r7, [r1], #1 + bgt L_aes_ctr_encrypt_arm32_crypto_192_start_byte + vrev32.8 q0, q2 + str r5, [r4] +L_aes_ctr_encrypt_arm32_crypto_192_partial_done: +#endif /* !NO_AES_192 */ + b L_aes_ctr_encrypt_arm32_crypto_done + # AES_CTR_256 +L_aes_ctr_encrypt_arm32_crypto_start_256: +#ifndef NO_AES_256 + vldm.32 r12!, {q11-q13} + mov lr, #1 + cmp r4, #1 + blt L_aes_ctr_encrypt_arm32_crypto_256_done + beq L_aes_ctr_encrypt_arm32_crypto_256_start_1 + adds r8, r8, #1 + adcs r7, r7, #0 + adcs r6, r6, #0 + adc r5, r5, #0 + vmov d3, r7, r8 + vmov d2, r5, r6 + vrev32.8 q1, q1 +L_aes_ctr_encrypt_arm32_crypto_256_start_2: + aese.8 q0, q3 + aesmc.8 q0, q0 + aese.8 q1, q3 + aesmc.8 q1, q1 + adds r8, r8, #1 + aese.8 q0, q4 + aesmc.8 q0, q0 + aese.8 q1, q4 + aesmc.8 q1, q1 + adcs r7, r7, #0 + aese.8 q0, q5 + aesmc.8 q0, q0 + aese.8 q1, q5 + aesmc.8 q1, q1 + adcs r6, r6, #0 + aese.8 q0, q6 + aesmc.8 q0, q0 + aese.8 q1, q6 + aesmc.8 q1, q1 + adc r5, r5, #0 + aese.8 q0, q7 + aesmc.8 q0, q0 + aese.8 q1, q7 + aesmc.8 q1, q1 + aese.8 q0, q8 + aesmc.8 q0, q0 + aese.8 q1, q8 + aesmc.8 q1, q1 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q1, q9 + aesmc.8 q1, q1 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q1, q10 + aesmc.8 q1, q1 + aese.8 q0, q11 + aesmc.8 q0, q0 + aese.8 q1, q11 + aesmc.8 q1, q1 + aese.8 q0, q12 + aesmc.8 q0, q0 + aese.8 q1, q12 + aesmc.8 q1, q1 + vld1.32 {q14}, [r12]! + vmov d5, r7, r8 + aese.8 q0, q13 + aesmc.8 q0, q0 + aese.8 q1, q13 + aesmc.8 q1, q1 + vld1.32 {q15}, [r12]! + vmov d4, r5, r6 + aese.8 q0, q14 + aesmc.8 q0, q0 + aese.8 q1, q14 + aesmc.8 q1, q1 + vld1.32 {q14}, [r12]! + adds r8, r8, #1 + aese.8 q0, q15 + aesmc.8 q0, q0 + aese.8 q1, q15 + aesmc.8 q1, q1 + vld1.32 {q15}, [r12] + adcs r7, r7, #0 + aese.8 q0, q14 + aese.8 q1, q14 + sub r4, r4, #2 + veor.32 q0, q0, q15 + veor.32 q1, q1, q15 + adcs r6, r6, #0 + vld1.8 {q14-q15}, [r0]! + sub r12, r12, #48 + veor.32 q14, q14, q0 + veor.32 q15, q15, q1 + adc r5, r5, #0 + vrev32.8 q0, q2 + vmov d2, r5, r6 + vmov d3, r7, r8 + cmp r4, #1 + vst1.8 {q14-q15}, [r1]! + vrev32.8 q1, q1 + bgt L_aes_ctr_encrypt_arm32_crypto_256_start_2 + mov lr, #0 + blt L_aes_ctr_encrypt_arm32_crypto_256_done +L_aes_ctr_encrypt_arm32_crypto_256_start_1: + aese.8 q0, q3 + aesmc.8 q0, q0 + adds r8, r8, lr + aese.8 q0, q4 + aesmc.8 q0, q0 + adcs r7, r7, #0 + aese.8 q0, q5 + aesmc.8 q0, q0 + adcs r6, r6, #0 + aese.8 q0, q6 + aesmc.8 q0, q0 + adc r5, r5, #0 + aese.8 q0, q7 + aesmc.8 q0, q0 + vmov d5, r7, r8 + aese.8 q0, q8 + aesmc.8 q0, q0 + vmov d4, r5, r6 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q0, q11 + aesmc.8 q0, q0 + aese.8 q0, q12 + aesmc.8 q0, q0 + vld1.32 {q14}, [r12]! + aese.8 q0, q13 + aesmc.8 q0, q0 + vld1.32 {q15}, [r12]! + aese.8 q0, q14 + aesmc.8 q0, q0 + vld1.32 {q14}, [r12]! + aese.8 q0, q15 + aesmc.8 q0, q0 + vld1.32 {q15}, [r12] + aese.8 q0, q14 + vld1.8 {q14}, [r0]! + veor.32 q0, q0, q15 + veor.32 q14, q14, q0 + sub r12, r12, #48 + vst1.8 {q14}, [r1]! + mov lr, #1 + vrev32.8 q0, q2 +L_aes_ctr_encrypt_arm32_crypto_256_done: + cmp r2, #0 + beq L_aes_ctr_encrypt_arm32_crypto_256_partial_done + ldr r4, [sp, #96] + aese.8 q0, q3 + aesmc.8 q0, q0 + adds r8, r8, lr + aese.8 q0, q4 + aesmc.8 q0, q0 + adcs r7, r7, #0 + aese.8 q0, q5 + aesmc.8 q0, q0 + adcs r6, r6, #0 + aese.8 q0, q6 + aesmc.8 q0, q0 + adc r5, r5, #0 + aese.8 q0, q7 + aesmc.8 q0, q0 + vmov d5, r7, r8 + aese.8 q0, q8 + aesmc.8 q0, q0 + vmov d4, r5, r6 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q0, q11 + aesmc.8 q0, q0 + aese.8 q0, q12 + aesmc.8 q0, q0 + vld1.32 {q14}, [r12]! + aese.8 q0, q13 + aesmc.8 q0, q0 + vld1.32 {q15}, [r12]! + aese.8 q0, q14 + aesmc.8 q0, q0 + vld1.32 {q14}, [r12]! + aese.8 q0, q15 + aesmc.8 q0, q0 + vld1.32 {q15}, [r12] + ldr lr, [sp, #92] + aese.8 q0, q14 + veor.32 q0, q0, q15 + vst1.32 {q0}, [lr] + vmov q0, q2 + mov r5, #16 + sub r5, r5, r2 +L_aes_ctr_encrypt_arm32_crypto_256_start_byte: + ldrb r7, [lr], #1 + ldrb r8, [r0], #1 + eor r7, r7, r8 + subs r2, r2, #1 + strb r7, [r1], #1 + bgt L_aes_ctr_encrypt_arm32_crypto_256_start_byte + vrev32.8 q0, q2 + str r5, [r4] +L_aes_ctr_encrypt_arm32_crypto_256_partial_done: +#endif /* !NO_AES_256 */ + b L_aes_ctr_encrypt_arm32_crypto_done + # AES_CTR_128 +L_aes_ctr_encrypt_arm32_crypto_start_128: +#ifndef NO_AES_128 + vldm.32 r12!, {q11-q13} + mov lr, #1 + cmp r4, #1 + blt L_aes_ctr_encrypt_arm32_crypto_128_done + beq L_aes_ctr_encrypt_arm32_crypto_128_start_1 + adds r8, r8, #1 + adcs r7, r7, #0 + adcs r6, r6, #0 + adc r5, r5, #0 + vmov d3, r7, r8 + vmov d2, r5, r6 + vrev32.8 q1, q1 +L_aes_ctr_encrypt_arm32_crypto_128_start_2: + aese.8 q0, q3 + aesmc.8 q0, q0 + aese.8 q1, q3 + aesmc.8 q1, q1 + adds r8, r8, #1 + aese.8 q0, q4 + aesmc.8 q0, q0 + aese.8 q1, q4 + aesmc.8 q1, q1 + adcs r7, r7, #0 + aese.8 q0, q5 + aesmc.8 q0, q0 + aese.8 q1, q5 + aesmc.8 q1, q1 + adcs r6, r6, #0 + aese.8 q0, q6 + aesmc.8 q0, q0 + aese.8 q1, q6 + aesmc.8 q1, q1 + adc r5, r5, #0 + aese.8 q0, q7 + aesmc.8 q0, q0 + aese.8 q1, q7 + aesmc.8 q1, q1 + vmov d5, r7, r8 + aese.8 q0, q8 + aesmc.8 q0, q0 + aese.8 q1, q8 + aesmc.8 q1, q1 + vmov d4, r5, r6 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q1, q9 + aesmc.8 q1, q1 + adds r8, r8, #1 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q1, q10 + aesmc.8 q1, q1 + adcs r7, r7, #0 + aese.8 q0, q11 + aesmc.8 q0, q0 + aese.8 q1, q11 + aesmc.8 q1, q1 + adcs r6, r6, #0 + vld1.8 {q14-q15}, [r0]! + adc r5, r5, #0 + aese.8 q0, q12 + aese.8 q1, q12 + sub r4, r4, #2 + veor.32 q14, q14, q13 + veor.32 q15, q15, q13 + veor.32 q14, q14, q0 + veor.32 q15, q15, q1 + vrev32.8 q0, q2 + vmov d2, r5, r6 + vmov d3, r7, r8 + cmp r4, #1 + vst1.8 {q14-q15}, [r1]! + vrev32.8 q1, q1 + bgt L_aes_ctr_encrypt_arm32_crypto_128_start_2 + mov lr, #0 + blt L_aes_ctr_encrypt_arm32_crypto_128_done +L_aes_ctr_encrypt_arm32_crypto_128_start_1: + aese.8 q0, q3 + aesmc.8 q0, q0 + adds r8, r8, lr + aese.8 q0, q4 + aesmc.8 q0, q0 + adcs r7, r7, #0 + aese.8 q0, q5 + aesmc.8 q0, q0 + adcs r6, r6, #0 + aese.8 q0, q6 + aesmc.8 q0, q0 + adc r5, r5, #0 + aese.8 q0, q7 + aesmc.8 q0, q0 + vmov d5, r7, r8 + aese.8 q0, q8 + aesmc.8 q0, q0 + vmov d4, r5, r6 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q0, q11 + aesmc.8 q0, q0 + vld1.8 {q14}, [r0]! + aese.8 q0, q12 + veor.32 q0, q0, q13 + veor.32 q14, q14, q0 + vst1.8 {q14}, [r1]! + mov lr, #1 + vrev32.8 q0, q2 +L_aes_ctr_encrypt_arm32_crypto_128_done: + cmp r2, #0 + beq L_aes_ctr_encrypt_arm32_crypto_128_partial_done + ldr r4, [sp, #96] + aese.8 q0, q3 + aesmc.8 q0, q0 + adds r8, r8, lr + aese.8 q0, q4 + aesmc.8 q0, q0 + adcs r7, r7, #0 + aese.8 q0, q5 + aesmc.8 q0, q0 + adcs r6, r6, #0 + aese.8 q0, q6 + aesmc.8 q0, q0 + adc r5, r5, #0 + aese.8 q0, q7 + aesmc.8 q0, q0 + vmov d5, r7, r8 + aese.8 q0, q8 + aesmc.8 q0, q0 + vmov d4, r5, r6 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q0, q11 + aesmc.8 q0, q0 + ldr lr, [sp, #92] + aese.8 q0, q12 + veor.32 q0, q0, q13 + vst1.32 {q0}, [lr] + vmov q0, q2 + mov r5, #16 + sub r5, r5, r2 +L_aes_ctr_encrypt_arm32_crypto_128_start_byte: + ldrb r7, [lr], #1 + ldrb r8, [r0], #1 + eor r7, r7, r8 + subs r2, r2, #1 + strb r7, [r1], #1 + bgt L_aes_ctr_encrypt_arm32_crypto_128_start_byte + vrev32.8 q0, q2 + str r5, [r4] +L_aes_ctr_encrypt_arm32_crypto_128_partial_done: +#endif /* !NO_AES_128 */ +L_aes_ctr_encrypt_arm32_crypto_done: + vst1.32 {q0}, [r3] + vpop {d8-d15} + pop {r4, r5, r6, r7, r8, pc} + .size AES_CTR_encrypt_AARCH32,.-AES_CTR_encrypt_AARCH32 +#endif /* WOLFSSL_AES_COUNTER */ +#ifdef HAVE_AESGCM + .text + .align 4 + .globl AES_GCM_set_key_AARCH32 + .type AES_GCM_set_key_AARCH32, %function +AES_GCM_set_key_AARCH32: + vpush {d8-d9} + vld1.8 {q0}, [r0] + vld1.8 {q1-q2}, [r1]! + vld1.8 {q3-q4}, [r1]! + aese.8 q0, q1 + aesmc.8 q0, q0 + aese.8 q0, q2 + aesmc.8 q0, q0 + aese.8 q0, q3 + aesmc.8 q0, q0 + aese.8 q0, q4 + aesmc.8 q0, q0 + vld1.8 {q1-q2}, [r1]! + vld1.8 {q3-q4}, [r1]! + aese.8 q0, q1 + aesmc.8 q0, q0 + aese.8 q0, q2 + aesmc.8 q0, q0 + aese.8 q0, q3 + aesmc.8 q0, q0 + aese.8 q0, q4 + aesmc.8 q0, q0 + subs r3, r3, #10 + vld1.8 {q1-q2}, [r1]! + aese.8 q0, q1 + aesmc.8 q0, q0 + aese.8 q0, q2 + beq L_aes_gcm_set_key_arm32_crypto_round_done + vld1.8 {q1-q2}, [r1]! + subs r3, r3, #2 + aesmc.8 q0, q0 + aese.8 q0, q1 + aesmc.8 q0, q0 + aese.8 q0, q2 + beq L_aes_gcm_set_key_arm32_crypto_round_done + vld1.8 {q1-q2}, [r1]! + aesmc.8 q0, q0 + aese.8 q0, q1 + aesmc.8 q0, q0 + aese.8 q0, q2 +L_aes_gcm_set_key_arm32_crypto_round_done: + vld1.8 {q1}, [r1] + veor q0, q0, q1 + vmov.i8 q1, #0x55 + vshl.u8 q2, q0, #1 + vshr.u8 q0, q0, #1 + vbif.8 q0, q2, q1 + vmov.i8 q1, #51 + vshl.u8 q2, q0, #2 + vshr.u8 q0, q0, #2 + vbit.8 q2, q0, q1 + vshl.u8 q0, q2, #4 + vsri.u8 q0, q2, #4 + vst1.32 {q0}, [r2] + vpop {d8-d9} + bx lr + .size AES_GCM_set_key_AARCH32,.-AES_GCM_set_key_AARCH32 + .text + .align 4 + .globl AES_GCM_encrypt_AARCH32 + .type AES_GCM_encrypt_AARCH32, %function +AES_GCM_encrypt_AARCH32: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + vpush {d8-d15} + # key + ldr r7, [sp, #120] + # tmp + ldr r9, [sp, #128] + # nonceSz + ldr r12, [sp, #100] + cmp r12, #12 + beq L_aes_gcm_encrypt_arm32_crypto_nonce_setup_done + vmov.i8 q13, #0x87 + # gcm_h + ldr r8, [sp, #124] + veor.8 q6, q6, q6 + vshr.u64 q13, q13, #56 + vld1.32 {q8}, [r8] +L_aes_gcm_encrypt_arm32_crypto_nonce_setup_done: + # Load Nonce + cmp r12, #12 + bne L_aes_gcm_encrypt_arm32_crypto_ghash_nonce + ldr r5, [r3] + ldr r8, [r3, #4] + ldr r12, [r3, #8] + vmov.i32 q6, #0x1000000 + vmov.32 s24, r5 + vmov.32 s25, r8 + vmov.32 s26, r12 + mov r5, #1 + b L_aes_gcm_encrypt_arm32_crypto_done_nonce +L_aes_gcm_encrypt_arm32_crypto_ghash_nonce: + lsr r10, r12, #4 + cmp r10, #0 + beq L_aes_gcm_encrypt_arm32_crypto_nonce_done +L_aes_gcm_encrypt_arm32_crypto_nonce_start_1: + vld1.32 {q14}, [r3]! + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q6, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q6, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q6 + # Reduce + vmull.p64 q6, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d13 + veor.8 d1, d1, d12 + vmull.p64 q6, d2, d26 + veor.8 q6, q6, q0 + # Done GHASH + subs r10, r10, #1 + bne L_aes_gcm_encrypt_arm32_crypto_nonce_start_1 +L_aes_gcm_encrypt_arm32_crypto_nonce_done: + ands r11, r12, #15 + beq L_aes_gcm_encrypt_arm32_crypto_nonce_partial_done + veor.8 q0, q0, q0 + mov r12, r11 + vst1.32 {q0}, [r9] + cmp r12, #4 + blt L_aes_gcm_encrypt_arm32_crypto_nonce_start_sw +L_aes_gcm_encrypt_arm32_crypto_nonce_start_dw: + ldr r8, [r3], #4 + sub r12, r12, #4 + str r8, [r9], #4 + cmp r12, #4 + bge L_aes_gcm_encrypt_arm32_crypto_nonce_start_dw +L_aes_gcm_encrypt_arm32_crypto_nonce_start_sw: + cmp r12, #2 + blt L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte + ldrh r8, [r3], #2 + sub r12, r12, #2 + strh r8, [r9], #2 +L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte: + cmp r12, #1 + blt L_aes_gcm_encrypt_arm32_crypto_nonce_end_bytes + ldrb r8, [r3], #1 + subs r12, r12, #1 + strb r8, [r9], #1 + bne L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte +L_aes_gcm_encrypt_arm32_crypto_nonce_end_bytes: + sub r9, r9, r11 + vld1.32 {q14}, [r9] + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q6, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q6, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q6 + # Reduce + vmull.p64 q6, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d13 + veor.8 d1, d1, d12 + vmull.p64 q6, d2, d26 + veor.8 q6, q6, q0 + # Done GHASH +L_aes_gcm_encrypt_arm32_crypto_nonce_partial_done: + veor.8 q0, q0, q0 + # nonceSz + ldr r12, [sp, #100] + lsr r10, r12, #29 + lsl r11, r12, #3 + rbit r10, r10 + rbit r11, r11 + vmov.32 s2, r10 + vmov.32 s3, r11 + veor.8 q6, q6, q0 + vmull.p64 q2, d13, d16 + vmull.p64 q12, d12, d17 + vmull.p64 q0, d12, d16 + vmull.p64 q1, d13, d17 + veor.8 q2, q2, q12 + vmull.p64 q6, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d13 + veor.8 d1, d1, d12 + vmull.p64 q6, d2, d26 + veor.8 q6, q6, q0 + vmov.32 r5, s24 + vmov.32 r8, s25 + vmov.32 r12, s26 + rbit r5, r5 + rbit r8, r8 + rbit r12, r12 + rev r5, r5 + rev r8, r8 + rev r12, r12 + vmov.32 s24, r5 + vmov.32 s25, r8 + vmov.32 s26, r12 + vmov.32 r5, s27 + rbit r5, r5 + rev r5, r5 + vmov.32 s27, r5 + rev r5, r5 +L_aes_gcm_encrypt_arm32_crypto_done_nonce: + vldm.32 r7!, {q0-q3} + vldm.32 r7!, {q7-q13} + # nr + ldr r12, [sp, #136] + lsr r10, r2, #4 + cmp r12, #12 + blt L_aes_gcm_encrypt_arm32_crypto_start_128 + bgt L_aes_gcm_encrypt_arm32_crypto_start_256 + # AES_GCM_192 +#ifndef NO_AES_192 + cmp r10, #1 + blt L_aes_gcm_encrypt_arm32_crypto_192_done + beq L_aes_gcm_encrypt_arm32_crypto_192_start_1 +L_aes_gcm_encrypt_arm32_crypto_192_start_2: + add r8, r5, #1 + vmov.8 q4, q6 + add r5, r5, #2 + vmov.8 q5, q6 + rev r8, r8 + rev r12, r5 + vmov s19, r8 + vmov s23, r12 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q5, q0 + aesmc.8 q5, q5 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q5, q1 + aesmc.8 q5, q5 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q5, q2 + aesmc.8 q5, q5 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q5, q3 + aesmc.8 q5, q5 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q5, q7 + aesmc.8 q5, q5 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q5, q8 + aesmc.8 q5, q5 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q5, q9 + aesmc.8 q5, q5 + subs r10, r10, #2 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q5, q10 + aesmc.8 q5, q5 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q5, q11 + aesmc.8 q5, q5 + aese.8 q4, q12 + aesmc.8 q4, q4 + aese.8 q5, q12 + aesmc.8 q5, q5 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + aese.8 q5, q13 + aesmc.8 q5, q5 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + aese.8 q5, q14 + veor.8 q5, q5, q15 + vld1.8 {q14-q15}, [r0]! + sub r7, r7, #16 + veor.8 q14, q14, q4 + veor.8 q15, q15, q5 + vst1.8 {q14-q15}, [r1]! + cmp r10, #1 + bgt L_aes_gcm_encrypt_arm32_crypto_192_start_2 + blt L_aes_gcm_encrypt_arm32_crypto_192_done +L_aes_gcm_encrypt_arm32_crypto_192_start_1: + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q4, q12 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + vld1.8 {q14}, [r0]! + sub r7, r7, #16 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r1]! +L_aes_gcm_encrypt_arm32_crypto_192_done: + ands r11, r2, #15 + beq L_aes_gcm_encrypt_arm32_crypto_192_partial_done + veor.8 q14, q14, q14 + mov r4, r11 + vst1.32 {q14}, [r9] + cmp r4, #4 + blt L_aes_gcm_encrypt_arm32_crypto_192_start_sw +L_aes_gcm_encrypt_arm32_crypto_192_start_dw: + ldr lr, [r0], #4 + sub r4, r4, #4 + str lr, [r9], #4 + cmp r4, #4 + bge L_aes_gcm_encrypt_arm32_crypto_192_start_dw +L_aes_gcm_encrypt_arm32_crypto_192_start_sw: + cmp r4, #2 + blt L_aes_gcm_encrypt_arm32_crypto_192_start_byte + ldrh lr, [r0], #2 + sub r4, r4, #2 + strh lr, [r9], #2 +L_aes_gcm_encrypt_arm32_crypto_192_start_byte: + cmp r4, #1 + blt L_aes_gcm_encrypt_arm32_crypto_192_end_bytes + ldrb lr, [r0], #1 + subs r4, r4, #1 + strb lr, [r9], #1 + bne L_aes_gcm_encrypt_arm32_crypto_192_start_byte +L_aes_gcm_encrypt_arm32_crypto_192_end_bytes: + sub r9, r9, r11 + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q4, q12 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + vld1.8 {q14}, [r9] + sub r7, r7, #16 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r9] + mov r4, r11 + cmp r4, #4 + blt L_aes_gcm_encrypt_arm32_crypto_192_out_start_sw +L_aes_gcm_encrypt_arm32_crypto_192_out_start_dw: + ldr lr, [r9], #4 + sub r4, r4, #4 + str lr, [r1], #4 + cmp r4, #4 + bge L_aes_gcm_encrypt_arm32_crypto_192_out_start_dw +L_aes_gcm_encrypt_arm32_crypto_192_out_start_sw: + cmp r4, #2 + blt L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte + ldrh lr, [r9], #2 + sub r4, r4, #2 + strh lr, [r1], #2 +L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte: + cmp r4, #1 + blt L_aes_gcm_encrypt_arm32_crypto_192_out_end_bytes + ldrb lr, [r9], #1 + subs r4, r4, #1 + strb lr, [r1], #1 + bne L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte +L_aes_gcm_encrypt_arm32_crypto_192_out_end_bytes: +L_aes_gcm_encrypt_arm32_crypto_192_partial_done: + # Finish + add r8, r2, #15 + sub r8, r5, r8, lsr #4 + rev r8, r8 + vmov.32 s27, r8 + aese.8 q6, q0 + aesmc.8 q6, q6 + aese.8 q6, q1 + aesmc.8 q6, q6 + aese.8 q6, q2 + aesmc.8 q6, q6 + aese.8 q6, q3 + aesmc.8 q6, q6 + aese.8 q6, q7 + aesmc.8 q6, q6 + aese.8 q6, q8 + aesmc.8 q6, q6 + aese.8 q6, q9 + aesmc.8 q6, q6 + aese.8 q6, q10 + aesmc.8 q6, q6 + aese.8 q6, q11 + aesmc.8 q6, q6 + aese.8 q6, q12 + aesmc.8 q6, q6 + vld1.32 {q14}, [r7]! + aese.8 q6, q13 + aesmc.8 q6, q6 + vld1.32 {q15}, [r7] + aese.8 q6, q14 + veor.8 q6, q6, q15 + sub r7, r7, #16 +#endif /* !NO_AES_192 */ + b L_aes_gcm_encrypt_arm32_crypto_done_enc + # AES_GCM_256 +L_aes_gcm_encrypt_arm32_crypto_start_256: +#ifndef NO_AES_256 + cmp r10, #1 + blt L_aes_gcm_encrypt_arm32_crypto_256_done + beq L_aes_gcm_encrypt_arm32_crypto_256_start_1 +L_aes_gcm_encrypt_arm32_crypto_256_start_2: + add r8, r5, #1 + vmov.8 q4, q6 + add r5, r5, #2 + vmov.8 q5, q6 + rev r8, r8 + rev r12, r5 + vmov s19, r8 + vmov s23, r12 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q5, q0 + aesmc.8 q5, q5 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q5, q1 + aesmc.8 q5, q5 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q5, q2 + aesmc.8 q5, q5 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q5, q3 + aesmc.8 q5, q5 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q5, q7 + aesmc.8 q5, q5 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q5, q8 + aesmc.8 q5, q5 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q5, q9 + aesmc.8 q5, q5 + subs r10, r10, #2 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q5, q10 + aesmc.8 q5, q5 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q5, q11 + aesmc.8 q5, q5 + aese.8 q4, q12 + aesmc.8 q4, q4 + aese.8 q5, q12 + aesmc.8 q5, q5 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + aese.8 q5, q13 + aesmc.8 q5, q5 + vld1.32 {q15}, [r7]! + aese.8 q4, q14 + aesmc.8 q4, q4 + aese.8 q5, q14 + aesmc.8 q5, q5 + vld1.32 {q14}, [r7]! + aese.8 q4, q15 + aesmc.8 q4, q4 + aese.8 q5, q15 + aesmc.8 q5, q5 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + aese.8 q5, q14 + veor.8 q5, q5, q15 + vld1.8 {q14-q15}, [r0]! + sub r7, r7, #48 + veor.8 q14, q14, q4 + veor.8 q15, q15, q5 + vst1.8 {q14-q15}, [r1]! + cmp r10, #1 + bgt L_aes_gcm_encrypt_arm32_crypto_256_start_2 + blt L_aes_gcm_encrypt_arm32_crypto_256_done +L_aes_gcm_encrypt_arm32_crypto_256_start_1: + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q4, q12 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7]! + aese.8 q4, q14 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q15 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + vld1.8 {q14}, [r0]! + sub r7, r7, #48 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r1]! +L_aes_gcm_encrypt_arm32_crypto_256_done: + ands r11, r2, #15 + beq L_aes_gcm_encrypt_arm32_crypto_256_partial_done + veor.8 q14, q14, q14 + mov r4, r11 + vst1.32 {q14}, [r9] + cmp r4, #4 + blt L_aes_gcm_encrypt_arm32_crypto_256_start_sw +L_aes_gcm_encrypt_arm32_crypto_256_start_dw: + ldr lr, [r0], #4 + sub r4, r4, #4 + str lr, [r9], #4 + cmp r4, #4 + bge L_aes_gcm_encrypt_arm32_crypto_256_start_dw +L_aes_gcm_encrypt_arm32_crypto_256_start_sw: + cmp r4, #2 + blt L_aes_gcm_encrypt_arm32_crypto_256_start_byte + ldrh lr, [r0], #2 + sub r4, r4, #2 + strh lr, [r9], #2 +L_aes_gcm_encrypt_arm32_crypto_256_start_byte: + cmp r4, #1 + blt L_aes_gcm_encrypt_arm32_crypto_256_end_bytes + ldrb lr, [r0], #1 + subs r4, r4, #1 + strb lr, [r9], #1 + bne L_aes_gcm_encrypt_arm32_crypto_256_start_byte +L_aes_gcm_encrypt_arm32_crypto_256_end_bytes: + sub r9, r9, r11 + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q4, q12 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7]! + aese.8 q4, q14 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q15 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + vld1.8 {q14}, [r9] + sub r7, r7, #48 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r9] + mov r4, r11 + cmp r4, #4 + blt L_aes_gcm_encrypt_arm32_crypto_256_out_start_sw +L_aes_gcm_encrypt_arm32_crypto_256_out_start_dw: + ldr lr, [r9], #4 + sub r4, r4, #4 + str lr, [r1], #4 + cmp r4, #4 + bge L_aes_gcm_encrypt_arm32_crypto_256_out_start_dw +L_aes_gcm_encrypt_arm32_crypto_256_out_start_sw: + cmp r4, #2 + blt L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte + ldrh lr, [r9], #2 + sub r4, r4, #2 + strh lr, [r1], #2 +L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte: + cmp r4, #1 + blt L_aes_gcm_encrypt_arm32_crypto_256_out_end_bytes + ldrb lr, [r9], #1 + subs r4, r4, #1 + strb lr, [r1], #1 + bne L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte +L_aes_gcm_encrypt_arm32_crypto_256_out_end_bytes: +L_aes_gcm_encrypt_arm32_crypto_256_partial_done: + # Finish + add r8, r2, #15 + sub r8, r5, r8, lsr #4 + rev r8, r8 + vmov.32 s27, r8 + aese.8 q6, q0 + aesmc.8 q6, q6 + aese.8 q6, q1 + aesmc.8 q6, q6 + aese.8 q6, q2 + aesmc.8 q6, q6 + aese.8 q6, q3 + aesmc.8 q6, q6 + aese.8 q6, q7 + aesmc.8 q6, q6 + aese.8 q6, q8 + aesmc.8 q6, q6 + aese.8 q6, q9 + aesmc.8 q6, q6 + aese.8 q6, q10 + aesmc.8 q6, q6 + aese.8 q6, q11 + aesmc.8 q6, q6 + aese.8 q6, q12 + aesmc.8 q6, q6 + vld1.32 {q14}, [r7]! + aese.8 q6, q13 + aesmc.8 q6, q6 + vld1.32 {q15}, [r7]! + aese.8 q6, q14 + aesmc.8 q6, q6 + vld1.32 {q14}, [r7]! + aese.8 q6, q15 + aesmc.8 q6, q6 + vld1.32 {q15}, [r7] + aese.8 q6, q14 + veor.8 q6, q6, q15 + sub r7, r7, #48 +#endif /* !NO_AES_256 */ + b L_aes_gcm_encrypt_arm32_crypto_done_enc + # AES_GCM_128 +L_aes_gcm_encrypt_arm32_crypto_start_128: +#ifndef NO_AES_128 + cmp r10, #1 + blt L_aes_gcm_encrypt_arm32_crypto_128_done + beq L_aes_gcm_encrypt_arm32_crypto_128_start_1 +L_aes_gcm_encrypt_arm32_crypto_128_start_2: + add r8, r5, #1 + vmov.8 q4, q6 + add r5, r5, #2 + vmov.8 q5, q6 + rev r8, r8 + rev r12, r5 + vmov s19, r8 + vmov s23, r12 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q5, q0 + aesmc.8 q5, q5 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q5, q1 + aesmc.8 q5, q5 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q5, q2 + aesmc.8 q5, q5 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q5, q3 + aesmc.8 q5, q5 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q5, q7 + aesmc.8 q5, q5 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q5, q8 + aesmc.8 q5, q5 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q5, q9 + aesmc.8 q5, q5 + subs r10, r10, #2 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q5, q10 + aesmc.8 q5, q5 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q5, q11 + aesmc.8 q5, q5 + vld1.8 {q14-q15}, [r0]! + aese.8 q4, q12 + veor.8 q4, q4, q13 + aese.8 q5, q12 + veor.8 q5, q5, q13 + veor.8 q14, q14, q4 + veor.8 q15, q15, q5 + vst1.8 {q14-q15}, [r1]! + cmp r10, #1 + bgt L_aes_gcm_encrypt_arm32_crypto_128_start_2 + blt L_aes_gcm_encrypt_arm32_crypto_128_done +L_aes_gcm_encrypt_arm32_crypto_128_start_1: + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + vld1.8 {q14}, [r0]! + aese.8 q4, q12 + veor.8 q4, q4, q13 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r1]! +L_aes_gcm_encrypt_arm32_crypto_128_done: + ands r11, r2, #15 + beq L_aes_gcm_encrypt_arm32_crypto_128_partial_done + veor.8 q14, q14, q14 + mov r4, r11 + vst1.32 {q14}, [r9] + cmp r4, #4 + blt L_aes_gcm_encrypt_arm32_crypto_128_start_sw +L_aes_gcm_encrypt_arm32_crypto_128_start_dw: + ldr lr, [r0], #4 + sub r4, r4, #4 + str lr, [r9], #4 + cmp r4, #4 + bge L_aes_gcm_encrypt_arm32_crypto_128_start_dw +L_aes_gcm_encrypt_arm32_crypto_128_start_sw: + cmp r4, #2 + blt L_aes_gcm_encrypt_arm32_crypto_128_start_byte + ldrh lr, [r0], #2 + sub r4, r4, #2 + strh lr, [r9], #2 +L_aes_gcm_encrypt_arm32_crypto_128_start_byte: + cmp r4, #1 + blt L_aes_gcm_encrypt_arm32_crypto_128_end_bytes + ldrb lr, [r0], #1 + subs r4, r4, #1 + strb lr, [r9], #1 + bne L_aes_gcm_encrypt_arm32_crypto_128_start_byte +L_aes_gcm_encrypt_arm32_crypto_128_end_bytes: + sub r9, r9, r11 + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + vld1.8 {q14}, [r9] + aese.8 q4, q12 + veor.8 q4, q4, q13 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r9] + mov r4, r11 + cmp r4, #4 + blt L_aes_gcm_encrypt_arm32_crypto_128_out_start_sw +L_aes_gcm_encrypt_arm32_crypto_128_out_start_dw: + ldr lr, [r9], #4 + sub r4, r4, #4 + str lr, [r1], #4 + cmp r4, #4 + bge L_aes_gcm_encrypt_arm32_crypto_128_out_start_dw +L_aes_gcm_encrypt_arm32_crypto_128_out_start_sw: + cmp r4, #2 + blt L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte + ldrh lr, [r9], #2 + sub r4, r4, #2 + strh lr, [r1], #2 +L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte: + cmp r4, #1 + blt L_aes_gcm_encrypt_arm32_crypto_128_out_end_bytes + ldrb lr, [r9], #1 + subs r4, r4, #1 + strb lr, [r1], #1 + bne L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte +L_aes_gcm_encrypt_arm32_crypto_128_out_end_bytes: +L_aes_gcm_encrypt_arm32_crypto_128_partial_done: + # Finish + add r8, r2, #15 + sub r8, r5, r8, lsr #4 + rev r8, r8 + vmov.32 s27, r8 + aese.8 q6, q0 + aesmc.8 q6, q6 + aese.8 q6, q1 + aesmc.8 q6, q6 + aese.8 q6, q2 + aesmc.8 q6, q6 + aese.8 q6, q3 + aesmc.8 q6, q6 + aese.8 q6, q7 + aesmc.8 q6, q6 + aese.8 q6, q8 + aesmc.8 q6, q6 + aese.8 q6, q9 + aesmc.8 q6, q6 + aese.8 q6, q10 + aesmc.8 q6, q6 + aese.8 q6, q11 + aesmc.8 q6, q6 + aese.8 q6, q12 + veor.8 q6, q6, q13 +#endif /* !NO_AES_128 */ +L_aes_gcm_encrypt_arm32_crypto_done_enc: + # aadSz + ldr r6, [sp, #116] + # gcm_h + ldr r8, [sp, #124] + sub r1, r1, r2 + vmov.i8 q13, #0x87 + veor.8 q7, q7, q7 + vshr.u64 q13, q13, #56 + vld1.32 {q8}, [r8] + orr r10, r6, r2 + cmp r10, #32 + blt L_aes_gcm_encrypt_arm32_crypto_h_done + # Square H => H^2 + vmull.p64 q1, d17, d17 + vmull.p64 q0, d16, d16 + vmull.p64 q9, d3, d27 + veor.8 d2, d2, d19 + veor.8 d1, d1, d18 + vmull.p64 q9, d2, d26 + veor.8 q9, q9, q0 + cmp r10, #0x40 + blt L_aes_gcm_encrypt_arm32_crypto_h_done + # Multiply H and H^2 => H^3 + vmull.p64 q2, d17, d18 + vmull.p64 q3, d16, d19 + vmull.p64 q0, d16, d18 + vmull.p64 q1, d17, d19 + veor.8 q2, q2, q3 + # Reduce + vmull.p64 q10, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d21 + veor.8 d1, d1, d20 + vmull.p64 q10, d2, d26 + veor.8 q10, q10, q0 + # Square H^2 => H^4 + vmull.p64 q1, d19, d19 + vmull.p64 q0, d18, d18 + vmull.p64 q11, d3, d27 + veor.8 d2, d2, d23 + veor.8 d1, d1, d22 + vmull.p64 q11, d2, d26 + veor.8 q11, q11, q0 + # Done +L_aes_gcm_encrypt_arm32_crypto_h_done: + # aad + ldr r5, [sp, #112] + lsr r10, r6, #4 + cmp r10, #1 + blt L_aes_gcm_encrypt_arm32_crypto_aad_done + beq L_aes_gcm_encrypt_arm32_crypto_aad_start_1 + cmp r10, #4 + blt L_aes_gcm_encrypt_arm32_crypto_aad_start_2 +L_aes_gcm_encrypt_arm32_crypto_aad_start_4: + vldm r5!, {q0-q2} + vmov.i8 q12, #0x55 + vmov.i8 q5, #51 + vshl.u8 q14, q0, #1 + vshl.u8 q15, q1, #1 + vshl.u8 q4, q2, #1 + ldr lr, [r5], #4 + vshr.u8 q0, q0, #1 + vshr.u8 q1, q1, #1 + vshr.u8 q2, q2, #1 + ldr r4, [r5], #4 + vbit.8 q14, q0, q12 + vbit.8 q15, q1, q12 + vbit.8 q4, q2, q12 + ldr r8, [r5], #4 + vshl.u8 q0, q14, #2 + vshl.u8 q1, q15, #2 + vshl.u8 q2, q4, #2 + ldr r12, [r5], #4 + vshr.u8 q14, q14, #2 + vshr.u8 q15, q15, #2 + vshr.u8 q4, q4, #2 + rbit lr, lr + vbit.8 q0, q14, q5 + rbit r4, r4 + vbit.8 q1, q15, q5 + rbit r8, r8 + vbit.8 q2, q4, q5 + rbit r12, r12 + vshl.u8 q14, q0, #4 + rev lr, lr + vshl.u8 q15, q1, #4 + rev r4, r4 + vshl.u8 q4, q2, #4 + rev r8, r8 + vsri.u8 q14, q0, #4 + rev r12, r12 + vsri.u8 q15, q1, #4 + vsri.u8 q4, q2, #4 + vmov d10, lr, r4 + vmov d11, r8, r12 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d11, d16 + vmull.p64 q7, d10, d17 + vmull.p64 q0, d10, d16 + vmull.p64 q1, d11, d17 + veor.8 q2, q2, q7 + # X += C * H^2 + vmull.p64 q3, d8, d18 + vmull.p64 q7, d9, d19 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d9, d18 + vmull.p64 q7, d8, d19 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # X += C * H^3 + vmull.p64 q3, d30, d20 + vmull.p64 q7, d31, d21 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d31, d20 + vmull.p64 q7, d30, d21 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # X += C * H^4 + vmull.p64 q3, d24, d22 + vmull.p64 q7, d25, d23 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d25, d22 + vmull.p64 q7, d24, d23 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH + sub r10, r10, #4 + cmp r10, #4 + bge L_aes_gcm_encrypt_arm32_crypto_aad_start_4 + cmp r10, #1 + blt L_aes_gcm_encrypt_arm32_crypto_aad_done + beq L_aes_gcm_encrypt_arm32_crypto_aad_start_1 +L_aes_gcm_encrypt_arm32_crypto_aad_start_2: + vld1.32 {q14-q15}, [r5]! + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshl.u8 q1, q15, #1 + vshr.u8 q14, q14, #1 + vshr.u8 q15, q15, #1 + vbif.8 q14, q0, q12 + vbif.8 q15, q1, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshl.u8 q1, q15, #2 + vshr.u8 q14, q14, #2 + vshr.u8 q15, q15, #2 + vbit.8 q0, q14, q12 + vbit.8 q1, q15, q12 + vshl.u8 q14, q0, #4 + vshl.u8 q15, q1, #4 + vsri.u8 q14, q0, #4 + vsri.u8 q15, q1, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d31, d16 + vmull.p64 q7, d30, d17 + vmull.p64 q0, d30, d16 + vmull.p64 q1, d31, d17 + veor.8 q2, q2, q7 + # X += C * H^2 + vmull.p64 q3, d24, d18 + vmull.p64 q7, d25, d19 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d25, d18 + vmull.p64 q7, d24, d19 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH + sub r10, r10, #2 + cmp r10, #0 + beq L_aes_gcm_encrypt_arm32_crypto_aad_done +L_aes_gcm_encrypt_arm32_crypto_aad_start_1: + vld1.32 {q14}, [r5]! + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q7, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH +L_aes_gcm_encrypt_arm32_crypto_aad_done: + ands r11, r6, #15 + beq L_aes_gcm_encrypt_arm32_crypto_aad_partial_done + veor.8 q0, q0, q0 + mov r12, r11 + vst1.32 {q0}, [r9] + cmp r12, #4 + blt L_aes_gcm_encrypt_arm32_crypto_aad_start_sw +L_aes_gcm_encrypt_arm32_crypto_aad_start_dw: + ldr r8, [r5], #4 + sub r12, r12, #4 + str r8, [r9], #4 + cmp r12, #4 + bge L_aes_gcm_encrypt_arm32_crypto_aad_start_dw +L_aes_gcm_encrypt_arm32_crypto_aad_start_sw: + cmp r12, #2 + blt L_aes_gcm_encrypt_arm32_crypto_aad_start_byte + ldrh r8, [r5], #2 + sub r12, r12, #2 + strh r8, [r9], #2 +L_aes_gcm_encrypt_arm32_crypto_aad_start_byte: + cmp r12, #1 + blt L_aes_gcm_encrypt_arm32_crypto_aad_end_bytes + ldrb r8, [r5], #1 + subs r12, r12, #1 + strb r8, [r9], #1 + bne L_aes_gcm_encrypt_arm32_crypto_aad_start_byte +L_aes_gcm_encrypt_arm32_crypto_aad_end_bytes: + sub r9, r9, r11 + vld1.32 {q14}, [r9] + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q7, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH +L_aes_gcm_encrypt_arm32_crypto_aad_partial_done: + # out + lsr r10, r2, #4 + cmp r10, #1 + blt L_aes_gcm_encrypt_arm32_crypto_out_done + beq L_aes_gcm_encrypt_arm32_crypto_out_start_1 + cmp r10, #4 + blt L_aes_gcm_encrypt_arm32_crypto_out_start_2 +L_aes_gcm_encrypt_arm32_crypto_out_start_4: + vldm r1!, {q0-q2} + vmov.i8 q12, #0x55 + vmov.i8 q5, #51 + vshl.u8 q14, q0, #1 + vshl.u8 q15, q1, #1 + vshl.u8 q4, q2, #1 + ldr lr, [r1], #4 + vshr.u8 q0, q0, #1 + vshr.u8 q1, q1, #1 + vshr.u8 q2, q2, #1 + ldr r4, [r1], #4 + vbit.8 q14, q0, q12 + vbit.8 q15, q1, q12 + vbit.8 q4, q2, q12 + ldr r8, [r1], #4 + vshl.u8 q0, q14, #2 + vshl.u8 q1, q15, #2 + vshl.u8 q2, q4, #2 + ldr r12, [r1], #4 + vshr.u8 q14, q14, #2 + vshr.u8 q15, q15, #2 + vshr.u8 q4, q4, #2 + rbit lr, lr + vbit.8 q0, q14, q5 + rbit r4, r4 + vbit.8 q1, q15, q5 + rbit r8, r8 + vbit.8 q2, q4, q5 + rbit r12, r12 + vshl.u8 q14, q0, #4 + rev lr, lr + vshl.u8 q15, q1, #4 + rev r4, r4 + vshl.u8 q4, q2, #4 + rev r8, r8 + vsri.u8 q14, q0, #4 + rev r12, r12 + vsri.u8 q15, q1, #4 + vsri.u8 q4, q2, #4 + vmov d10, lr, r4 + vmov d11, r8, r12 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d11, d16 + vmull.p64 q7, d10, d17 + vmull.p64 q0, d10, d16 + vmull.p64 q1, d11, d17 + veor.8 q2, q2, q7 + # X += C * H^2 + vmull.p64 q3, d8, d18 + vmull.p64 q7, d9, d19 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d9, d18 + vmull.p64 q7, d8, d19 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # X += C * H^3 + vmull.p64 q3, d30, d20 + vmull.p64 q7, d31, d21 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d31, d20 + vmull.p64 q7, d30, d21 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # X += C * H^4 + vmull.p64 q3, d24, d22 + vmull.p64 q7, d25, d23 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d25, d22 + vmull.p64 q7, d24, d23 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH + sub r10, r10, #4 + cmp r10, #4 + bge L_aes_gcm_encrypt_arm32_crypto_out_start_4 + cmp r10, #1 + blt L_aes_gcm_encrypt_arm32_crypto_out_done + beq L_aes_gcm_encrypt_arm32_crypto_out_start_1 +L_aes_gcm_encrypt_arm32_crypto_out_start_2: + vld1.32 {q14-q15}, [r1]! + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshl.u8 q1, q15, #1 + vshr.u8 q14, q14, #1 + vshr.u8 q15, q15, #1 + vbif.8 q14, q0, q12 + vbif.8 q15, q1, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshl.u8 q1, q15, #2 + vshr.u8 q14, q14, #2 + vshr.u8 q15, q15, #2 + vbit.8 q0, q14, q12 + vbit.8 q1, q15, q12 + vshl.u8 q14, q0, #4 + vshl.u8 q15, q1, #4 + vsri.u8 q14, q0, #4 + vsri.u8 q15, q1, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d31, d16 + vmull.p64 q7, d30, d17 + vmull.p64 q0, d30, d16 + vmull.p64 q1, d31, d17 + veor.8 q2, q2, q7 + # X += C * H^2 + vmull.p64 q3, d24, d18 + vmull.p64 q7, d25, d19 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d25, d18 + vmull.p64 q7, d24, d19 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH + sub r10, r10, #2 + cmp r10, #0 + beq L_aes_gcm_encrypt_arm32_crypto_out_done +L_aes_gcm_encrypt_arm32_crypto_out_start_1: + vld1.32 {q14}, [r1]! + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q7, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH +L_aes_gcm_encrypt_arm32_crypto_out_done: + ands r11, r2, #15 + beq L_aes_gcm_encrypt_arm32_crypto_out_partial_done + veor.8 q0, q0, q0 + mov r12, r11 + vst1.32 {q0}, [r9] + cmp r12, #4 + blt L_aes_gcm_encrypt_arm32_crypto_out_start_sw +L_aes_gcm_encrypt_arm32_crypto_out_start_dw: + ldr r8, [r1], #4 + sub r12, r12, #4 + str r8, [r9], #4 + cmp r12, #4 + bge L_aes_gcm_encrypt_arm32_crypto_out_start_dw +L_aes_gcm_encrypt_arm32_crypto_out_start_sw: + cmp r12, #2 + blt L_aes_gcm_encrypt_arm32_crypto_out_start_byte + ldrh r8, [r1], #2 + sub r12, r12, #2 + strh r8, [r9], #2 +L_aes_gcm_encrypt_arm32_crypto_out_start_byte: + cmp r12, #1 + blt L_aes_gcm_encrypt_arm32_crypto_out_end_bytes + ldrb r8, [r1], #1 + subs r12, r12, #1 + strb r8, [r9], #1 + bne L_aes_gcm_encrypt_arm32_crypto_out_start_byte +L_aes_gcm_encrypt_arm32_crypto_out_end_bytes: + sub r9, r9, r11 + vld1.32 {q14}, [r9] + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q7, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH +L_aes_gcm_encrypt_arm32_crypto_out_partial_done: + lsr lr, r6, #29 + lsl r6, r6, #3 + rbit lr, lr + rbit r6, r6 + vmov s0, lr + vmov s1, r6 + lsr lr, r2, #29 + lsl r2, r2, #3 + rbit lr, lr + rbit r2, r2 + vmov s2, lr + vmov s3, r2 + veor.8 q7, q7, q0 + vmull.p64 q2, d15, d16 + vmull.p64 q3, d14, d17 + vmull.p64 q0, d14, d16 + vmull.p64 q1, d15, d17 + veor.8 q2, q2, q3 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + vmov.i8 q0, #0x55 + vshl.u8 q1, q7, #1 + vshr.u8 q7, q7, #1 + vbif.8 q7, q1, q0 + vmov.i8 q0, #51 + vshl.u8 q1, q7, #2 + vshr.u8 q7, q7, #2 + vbit.8 q1, q7, q0 + vshl.u8 q7, q1, #4 + vsri.u8 q7, q1, #4 + # DONE + veor.8 q7, q7, q6 + # tag + ldr lr, [sp, #104] + # tagSz + ldr r4, [sp, #108] + cmp r4, #16 + bne L_aes_gcm_encrypt_arm32_crypto_tag_tag_partial + vst1.8 {q7}, [lr] + b L_aes_gcm_encrypt_arm32_crypto_done_gcm +L_aes_gcm_encrypt_arm32_crypto_tag_tag_partial: + vst1.8 {q7}, [r9] + cmp r4, #4 + blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_sw +L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_dw: + ldr r8, [r9], #4 + sub r4, r4, #4 + str r8, [lr], #4 + cmp r4, #4 + bge L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_dw +L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_sw: + cmp r4, #2 + blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte + ldrh r8, [r9], #2 + sub r4, r4, #2 + strh r8, [lr], #2 +L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte: + cmp r4, #1 + blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_end_bytes + ldrb r8, [r9], #1 + subs r4, r4, #1 + strb r8, [lr], #1 + bne L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte +L_aes_gcm_encrypt_arm32_crypto_tag_tag_end_bytes: +L_aes_gcm_encrypt_arm32_crypto_done_gcm: + vpop {d8-d15} + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size AES_GCM_encrypt_AARCH32,.-AES_GCM_encrypt_AARCH32 +#ifdef HAVE_AES_DECRYPT + .text + .align 4 + .globl AES_GCM_decrypt_AARCH32 + .type AES_GCM_decrypt_AARCH32, %function +AES_GCM_decrypt_AARCH32: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + vpush {d8-d15} + # key + ldr r7, [sp, #120] + # tmp + ldr r9, [sp, #128] + # aadSz + ldr r6, [sp, #116] + # gcm_h + ldr r8, [sp, #124] + vmov.i8 q13, #0x87 + veor.8 q7, q7, q7 + vshr.u64 q13, q13, #56 + vld1.32 {q8}, [r8] + orr r10, r6, r2 + cmp r10, #32 + blt L_aes_gcm_decrypt_arm32_crypto_h_done + # Square H => H^2 + vmull.p64 q1, d17, d17 + vmull.p64 q0, d16, d16 + vmull.p64 q9, d3, d27 + veor.8 d2, d2, d19 + veor.8 d1, d1, d18 + vmull.p64 q9, d2, d26 + veor.8 q9, q9, q0 + cmp r10, #0x40 + blt L_aes_gcm_decrypt_arm32_crypto_h_done + # Multiply H and H^2 => H^3 + vmull.p64 q2, d17, d18 + vmull.p64 q3, d16, d19 + vmull.p64 q0, d16, d18 + vmull.p64 q1, d17, d19 + veor.8 q2, q2, q3 + # Reduce + vmull.p64 q10, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d21 + veor.8 d1, d1, d20 + vmull.p64 q10, d2, d26 + veor.8 q10, q10, q0 + # Square H^2 => H^4 + vmull.p64 q1, d19, d19 + vmull.p64 q0, d18, d18 + vmull.p64 q11, d3, d27 + veor.8 d2, d2, d23 + veor.8 d1, d1, d22 + vmull.p64 q11, d2, d26 + veor.8 q11, q11, q0 + # Done +L_aes_gcm_decrypt_arm32_crypto_h_done: + # aad + ldr r5, [sp, #112] + lsr r10, r6, #4 + cmp r10, #1 + blt L_aes_gcm_decrypt_arm32_crypto_aad_done + beq L_aes_gcm_decrypt_arm32_crypto_aad_start_1 + cmp r10, #4 + blt L_aes_gcm_decrypt_arm32_crypto_aad_start_2 +L_aes_gcm_decrypt_arm32_crypto_aad_start_4: + vldm r5!, {q0-q2} + vmov.i8 q12, #0x55 + vmov.i8 q5, #51 + vshl.u8 q14, q0, #1 + vshl.u8 q15, q1, #1 + vshl.u8 q4, q2, #1 + ldr lr, [r5], #4 + vshr.u8 q0, q0, #1 + vshr.u8 q1, q1, #1 + vshr.u8 q2, q2, #1 + ldr r4, [r5], #4 + vbit.8 q14, q0, q12 + vbit.8 q15, q1, q12 + vbit.8 q4, q2, q12 + ldr r8, [r5], #4 + vshl.u8 q0, q14, #2 + vshl.u8 q1, q15, #2 + vshl.u8 q2, q4, #2 + ldr r12, [r5], #4 + vshr.u8 q14, q14, #2 + vshr.u8 q15, q15, #2 + vshr.u8 q4, q4, #2 + rbit lr, lr + vbit.8 q0, q14, q5 + rbit r4, r4 + vbit.8 q1, q15, q5 + rbit r8, r8 + vbit.8 q2, q4, q5 + rbit r12, r12 + vshl.u8 q14, q0, #4 + rev lr, lr + vshl.u8 q15, q1, #4 + rev r4, r4 + vshl.u8 q4, q2, #4 + rev r8, r8 + vsri.u8 q14, q0, #4 + rev r12, r12 + vsri.u8 q15, q1, #4 + vsri.u8 q4, q2, #4 + vmov d10, lr, r4 + vmov d11, r8, r12 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d11, d16 + vmull.p64 q7, d10, d17 + vmull.p64 q0, d10, d16 + vmull.p64 q1, d11, d17 + veor.8 q2, q2, q7 + # X += C * H^2 + vmull.p64 q3, d8, d18 + vmull.p64 q7, d9, d19 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d9, d18 + vmull.p64 q7, d8, d19 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # X += C * H^3 + vmull.p64 q3, d30, d20 + vmull.p64 q7, d31, d21 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d31, d20 + vmull.p64 q7, d30, d21 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # X += C * H^4 + vmull.p64 q3, d24, d22 + vmull.p64 q7, d25, d23 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d25, d22 + vmull.p64 q7, d24, d23 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH + sub r10, r10, #4 + cmp r10, #4 + bge L_aes_gcm_decrypt_arm32_crypto_aad_start_4 + cmp r10, #1 + blt L_aes_gcm_decrypt_arm32_crypto_aad_done + beq L_aes_gcm_decrypt_arm32_crypto_aad_start_1 +L_aes_gcm_decrypt_arm32_crypto_aad_start_2: + vld1.32 {q14-q15}, [r5]! + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshl.u8 q1, q15, #1 + vshr.u8 q14, q14, #1 + vshr.u8 q15, q15, #1 + vbif.8 q14, q0, q12 + vbif.8 q15, q1, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshl.u8 q1, q15, #2 + vshr.u8 q14, q14, #2 + vshr.u8 q15, q15, #2 + vbit.8 q0, q14, q12 + vbit.8 q1, q15, q12 + vshl.u8 q14, q0, #4 + vshl.u8 q15, q1, #4 + vsri.u8 q14, q0, #4 + vsri.u8 q15, q1, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d31, d16 + vmull.p64 q7, d30, d17 + vmull.p64 q0, d30, d16 + vmull.p64 q1, d31, d17 + veor.8 q2, q2, q7 + # X += C * H^2 + vmull.p64 q3, d24, d18 + vmull.p64 q7, d25, d19 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d25, d18 + vmull.p64 q7, d24, d19 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH + sub r10, r10, #2 + cmp r10, #0 + beq L_aes_gcm_decrypt_arm32_crypto_aad_done +L_aes_gcm_decrypt_arm32_crypto_aad_start_1: + vld1.32 {q14}, [r5]! + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q7, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH +L_aes_gcm_decrypt_arm32_crypto_aad_done: + ands r11, r6, #15 + beq L_aes_gcm_decrypt_arm32_crypto_aad_partial_done + veor.8 q0, q0, q0 + mov r12, r11 + vst1.32 {q0}, [r9] + cmp r12, #4 + blt L_aes_gcm_decrypt_arm32_crypto_aad_start_sw +L_aes_gcm_decrypt_arm32_crypto_aad_start_dw: + ldr r8, [r5], #4 + sub r12, r12, #4 + str r8, [r9], #4 + cmp r12, #4 + bge L_aes_gcm_decrypt_arm32_crypto_aad_start_dw +L_aes_gcm_decrypt_arm32_crypto_aad_start_sw: + cmp r12, #2 + blt L_aes_gcm_decrypt_arm32_crypto_aad_start_byte + ldrh r8, [r5], #2 + sub r12, r12, #2 + strh r8, [r9], #2 +L_aes_gcm_decrypt_arm32_crypto_aad_start_byte: + cmp r12, #1 + blt L_aes_gcm_decrypt_arm32_crypto_aad_end_bytes + ldrb r8, [r5], #1 + subs r12, r12, #1 + strb r8, [r9], #1 + bne L_aes_gcm_decrypt_arm32_crypto_aad_start_byte +L_aes_gcm_decrypt_arm32_crypto_aad_end_bytes: + sub r9, r9, r11 + vld1.32 {q14}, [r9] + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q7, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH +L_aes_gcm_decrypt_arm32_crypto_aad_partial_done: + # in + lsr r10, r2, #4 + cmp r10, #1 + blt L_aes_gcm_decrypt_arm32_crypto_in_done + beq L_aes_gcm_decrypt_arm32_crypto_in_start_1 + cmp r10, #4 + blt L_aes_gcm_decrypt_arm32_crypto_in_start_2 +L_aes_gcm_decrypt_arm32_crypto_in_start_4: + vldm r0!, {q0-q2} + vmov.i8 q12, #0x55 + vmov.i8 q5, #51 + vshl.u8 q14, q0, #1 + vshl.u8 q15, q1, #1 + vshl.u8 q4, q2, #1 + ldr lr, [r0], #4 + vshr.u8 q0, q0, #1 + vshr.u8 q1, q1, #1 + vshr.u8 q2, q2, #1 + ldr r4, [r0], #4 + vbit.8 q14, q0, q12 + vbit.8 q15, q1, q12 + vbit.8 q4, q2, q12 + ldr r8, [r0], #4 + vshl.u8 q0, q14, #2 + vshl.u8 q1, q15, #2 + vshl.u8 q2, q4, #2 + ldr r12, [r0], #4 + vshr.u8 q14, q14, #2 + vshr.u8 q15, q15, #2 + vshr.u8 q4, q4, #2 + rbit lr, lr + vbit.8 q0, q14, q5 + rbit r4, r4 + vbit.8 q1, q15, q5 + rbit r8, r8 + vbit.8 q2, q4, q5 + rbit r12, r12 + vshl.u8 q14, q0, #4 + rev lr, lr + vshl.u8 q15, q1, #4 rev r4, r4 + vshl.u8 q4, q2, #4 + rev r8, r8 + vsri.u8 q14, q0, #4 + rev r12, r12 + vsri.u8 q15, q1, #4 + vsri.u8 q4, q2, #4 + vmov d10, lr, r4 + vmov d11, r8, r12 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d11, d16 + vmull.p64 q7, d10, d17 + vmull.p64 q0, d10, d16 + vmull.p64 q1, d11, d17 + veor.8 q2, q2, q7 + # X += C * H^2 + vmull.p64 q3, d8, d18 + vmull.p64 q7, d9, d19 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d9, d18 + vmull.p64 q7, d8, d19 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # X += C * H^3 + vmull.p64 q3, d30, d20 + vmull.p64 q7, d31, d21 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d31, d20 + vmull.p64 q7, d30, d21 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # X += C * H^4 + vmull.p64 q3, d24, d22 + vmull.p64 q7, d25, d23 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d25, d22 + vmull.p64 q7, d24, d23 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH + sub r10, r10, #4 + cmp r10, #4 + bge L_aes_gcm_decrypt_arm32_crypto_in_start_4 + cmp r10, #1 + blt L_aes_gcm_decrypt_arm32_crypto_in_done + beq L_aes_gcm_decrypt_arm32_crypto_in_start_1 +L_aes_gcm_decrypt_arm32_crypto_in_start_2: + vld1.32 {q14-q15}, [r0]! + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshl.u8 q1, q15, #1 + vshr.u8 q14, q14, #1 + vshr.u8 q15, q15, #1 + vbif.8 q14, q0, q12 + vbif.8 q15, q1, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshl.u8 q1, q15, #2 + vshr.u8 q14, q14, #2 + vshr.u8 q15, q15, #2 + vbit.8 q0, q14, q12 + vbit.8 q1, q15, q12 + vshl.u8 q14, q0, #4 + vshl.u8 q15, q1, #4 + vsri.u8 q14, q0, #4 + vsri.u8 q15, q1, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d31, d16 + vmull.p64 q7, d30, d17 + vmull.p64 q0, d30, d16 + vmull.p64 q1, d31, d17 + veor.8 q2, q2, q7 + # X += C * H^2 + vmull.p64 q3, d24, d18 + vmull.p64 q7, d25, d19 + veor.8 q0, q0, q3 + veor.8 q1, q1, q7 + vmull.p64 q3, d25, d18 + vmull.p64 q7, d24, d19 + veor.8 q2, q2, q3 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH + sub r10, r10, #2 + cmp r10, #0 + beq L_aes_gcm_decrypt_arm32_crypto_in_done +L_aes_gcm_decrypt_arm32_crypto_in_start_1: + vld1.32 {q14}, [r0]! + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q7, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH +L_aes_gcm_decrypt_arm32_crypto_in_done: + ands r11, r2, #15 + beq L_aes_gcm_decrypt_arm32_crypto_in_partial_done + veor.8 q0, q0, q0 + mov r12, r11 + vst1.32 {q0}, [r9] + cmp r12, #4 + blt L_aes_gcm_decrypt_arm32_crypto_in_start_sw +L_aes_gcm_decrypt_arm32_crypto_in_start_dw: + ldr r8, [r0], #4 + sub r12, r12, #4 + str r8, [r9], #4 + cmp r12, #4 + bge L_aes_gcm_decrypt_arm32_crypto_in_start_dw +L_aes_gcm_decrypt_arm32_crypto_in_start_sw: + cmp r12, #2 + blt L_aes_gcm_decrypt_arm32_crypto_in_start_byte + ldrh r8, [r0], #2 + sub r12, r12, #2 + strh r8, [r9], #2 +L_aes_gcm_decrypt_arm32_crypto_in_start_byte: + cmp r12, #1 + blt L_aes_gcm_decrypt_arm32_crypto_in_end_bytes + ldrb r8, [r0], #1 + subs r12, r12, #1 + strb r8, [r9], #1 + bne L_aes_gcm_decrypt_arm32_crypto_in_start_byte +L_aes_gcm_decrypt_arm32_crypto_in_end_bytes: + sub r9, r9, r11 + vld1.32 {q14}, [r9] + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q7, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q7, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q7 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + # Done GHASH +L_aes_gcm_decrypt_arm32_crypto_in_partial_done: + sub r0, r0, r2 +L_aes_gcm_decrypt_arm32_crypto_done_gcm: + # nonceSz + ldr r12, [sp, #100] + # Load Nonce + cmp r12, #12 + bne L_aes_gcm_decrypt_arm32_crypto_ghash_nonce + ldr r5, [r3] + ldr r8, [r3, #4] + ldr r12, [r3, #8] + vmov.i32 q6, #0x1000000 + vmov.32 s24, r5 + vmov.32 s25, r8 + vmov.32 s26, r12 + mov r5, #1 + b L_aes_gcm_decrypt_arm32_crypto_done_nonce +L_aes_gcm_decrypt_arm32_crypto_ghash_nonce: + lsr r10, r12, #4 + cmp r10, #0 + beq L_aes_gcm_decrypt_arm32_crypto_nonce_done +L_aes_gcm_decrypt_arm32_crypto_nonce_start_1: + vld1.32 {q14}, [r3]! + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q6, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q6, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q6 + # Reduce + vmull.p64 q6, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d13 + veor.8 d1, d1, d12 + vmull.p64 q6, d2, d26 + veor.8 q6, q6, q0 + # Done GHASH + subs r10, r10, #1 + bne L_aes_gcm_decrypt_arm32_crypto_nonce_start_1 +L_aes_gcm_decrypt_arm32_crypto_nonce_done: + ands r11, r12, #15 + beq L_aes_gcm_decrypt_arm32_crypto_nonce_partial_done + veor.8 q0, q0, q0 + mov r12, r11 + vst1.32 {q0}, [r9] + cmp r12, #4 + blt L_aes_gcm_decrypt_arm32_crypto_nonce_start_sw +L_aes_gcm_decrypt_arm32_crypto_nonce_start_dw: + ldr r8, [r3], #4 + sub r12, r12, #4 + str r8, [r9], #4 + cmp r12, #4 + bge L_aes_gcm_decrypt_arm32_crypto_nonce_start_dw +L_aes_gcm_decrypt_arm32_crypto_nonce_start_sw: + cmp r12, #2 + blt L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte + ldrh r8, [r3], #2 + sub r12, r12, #2 + strh r8, [r9], #2 +L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte: + cmp r12, #1 + blt L_aes_gcm_decrypt_arm32_crypto_nonce_end_bytes + ldrb r8, [r3], #1 + subs r12, r12, #1 + strb r8, [r9], #1 + bne L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte +L_aes_gcm_decrypt_arm32_crypto_nonce_end_bytes: + sub r9, r9, r11 + vld1.32 {q14}, [r9] + vmov.i8 q12, #0x55 + vshl.u8 q0, q14, #1 + vshr.u8 q14, q14, #1 + vbif.8 q14, q0, q12 + vmov.i8 q12, #51 + vshl.u8 q0, q14, #2 + vshr.u8 q14, q14, #2 + vbit.8 q0, q14, q12 + vshl.u8 q14, q0, #4 + vsri.u8 q14, q0, #4 + veor.8 q12, q6, q14 + # X = C * H^1 + vmull.p64 q2, d25, d16 + vmull.p64 q6, d24, d17 + vmull.p64 q0, d24, d16 + vmull.p64 q1, d25, d17 + veor.8 q2, q2, q6 + # Reduce + vmull.p64 q6, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d13 + veor.8 d1, d1, d12 + vmull.p64 q6, d2, d26 + veor.8 q6, q6, q0 + # Done GHASH +L_aes_gcm_decrypt_arm32_crypto_nonce_partial_done: + veor.8 q0, q0, q0 + # nonceSz + ldr r12, [sp, #100] + lsr r10, r12, #29 + lsl r11, r12, #3 + rbit r10, r10 + rbit r11, r11 + vmov.32 s2, r10 + vmov.32 s3, r11 + veor.8 q6, q6, q0 + vmull.p64 q2, d13, d16 + vmull.p64 q12, d12, d17 + vmull.p64 q0, d12, d16 + vmull.p64 q1, d13, d17 + veor.8 q2, q2, q12 + vmull.p64 q6, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d13 + veor.8 d1, d1, d12 + vmull.p64 q6, d2, d26 + veor.8 q6, q6, q0 + vmov.32 r5, s24 + vmov.32 r8, s25 + vmov.32 r12, s26 + rbit r5, r5 + rbit r8, r8 + rbit r12, r12 rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - ldr r8, [lr] - ldr r9, [lr, #4] - ldr r10, [lr, #8] - ldr r11, [lr, #12] - eor r4, r4, r8 - eor r5, r5, r9 - eor r6, r6, r10 - eor r7, r7, r11 - ldr r8, [sp, #4] - str r4, [r1] - str r5, [r1, #4] - str r6, [r1, #8] - str r7, [r1, #12] - ldm r8, {r4, r5, r6, r7} - subs r2, r2, #16 - add lr, lr, #16 + rev r8, r8 + rev r12, r12 + vmov.32 s24, r5 + vmov.32 s25, r8 + vmov.32 s26, r12 + vmov.32 r5, s27 + rbit r5, r5 + rev r5, r5 + vmov.32 s27, r5 + rev r5, r5 +L_aes_gcm_decrypt_arm32_crypto_done_nonce: + # reg + ldr r9, [sp, #132] + vst1.32 {q7}, [r9] + # tmp + ldr r9, [sp, #128] + vldm.32 r7!, {q0-q3} + vldm.32 r7!, {q7-q13} + # nr + ldr r12, [sp, #136] + lsr r10, r2, #4 + and r11, r2, #15 + cmp r12, #12 + blt L_aes_gcm_decrypt_arm32_crypto_start_128 + bgt L_aes_gcm_decrypt_arm32_crypto_start_256 + # AES_GCM_192 +#ifndef NO_AES_192 + cmp r10, #1 + blt L_aes_gcm_decrypt_arm32_crypto_192_done + beq L_aes_gcm_decrypt_arm32_crypto_192_start_1 +L_aes_gcm_decrypt_arm32_crypto_192_start_2: + add r8, r5, #1 + vmov.8 q4, q6 + add r5, r5, #2 + vmov.8 q5, q6 + rev r8, r8 + rev r12, r5 + vmov s19, r8 + vmov s23, r12 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q5, q0 + aesmc.8 q5, q5 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q5, q1 + aesmc.8 q5, q5 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q5, q2 + aesmc.8 q5, q5 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q5, q3 + aesmc.8 q5, q5 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q5, q7 + aesmc.8 q5, q5 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q5, q8 + aesmc.8 q5, q5 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q5, q9 + aesmc.8 q5, q5 + subs r10, r10, #2 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q5, q10 + aesmc.8 q5, q5 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q5, q11 + aesmc.8 q5, q5 + aese.8 q4, q12 + aesmc.8 q4, q4 + aese.8 q5, q12 + aesmc.8 q5, q5 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + aese.8 q5, q13 + aesmc.8 q5, q5 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + aese.8 q5, q14 + veor.8 q5, q5, q15 + vld1.8 {q14-q15}, [r0]! + sub r7, r7, #16 + veor.8 q14, q14, q4 + veor.8 q15, q15, q5 + vst1.8 {q14-q15}, [r1]! + cmp r10, #1 + bgt L_aes_gcm_decrypt_arm32_crypto_192_start_2 + blt L_aes_gcm_decrypt_arm32_crypto_192_done +L_aes_gcm_decrypt_arm32_crypto_192_start_1: + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q4, q12 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + vld1.8 {q14}, [r0]! + sub r7, r7, #16 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r1]! +L_aes_gcm_decrypt_arm32_crypto_192_done: + ands r11, r2, #15 + beq L_aes_gcm_decrypt_arm32_crypto_192_partial_done + veor.8 q14, q14, q14 + mov r4, r11 + vst1.32 {q14}, [r9] + cmp r4, #4 + blt L_aes_gcm_decrypt_arm32_crypto_192_start_sw +L_aes_gcm_decrypt_arm32_crypto_192_start_dw: + ldr lr, [r0], #4 + sub r4, r4, #4 + str lr, [r9], #4 + cmp r4, #4 + bge L_aes_gcm_decrypt_arm32_crypto_192_start_dw +L_aes_gcm_decrypt_arm32_crypto_192_start_sw: + cmp r4, #2 + blt L_aes_gcm_decrypt_arm32_crypto_192_start_byte + ldrh lr, [r0], #2 + sub r4, r4, #2 + strh lr, [r9], #2 +L_aes_gcm_decrypt_arm32_crypto_192_start_byte: + cmp r4, #1 + blt L_aes_gcm_decrypt_arm32_crypto_192_end_bytes + ldrb lr, [r0], #1 + subs r4, r4, #1 + strb lr, [r9], #1 + bne L_aes_gcm_decrypt_arm32_crypto_192_start_byte +L_aes_gcm_decrypt_arm32_crypto_192_end_bytes: + sub r9, r9, r11 + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q4, q12 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + vld1.8 {q14}, [r9] + sub r7, r7, #16 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r9] + mov r4, r11 + cmp r4, #4 + blt L_aes_gcm_decrypt_arm32_crypto_192_out_start_sw +L_aes_gcm_decrypt_arm32_crypto_192_out_start_dw: + ldr lr, [r9], #4 + sub r4, r4, #4 + str lr, [r1], #4 + cmp r4, #4 + bge L_aes_gcm_decrypt_arm32_crypto_192_out_start_dw +L_aes_gcm_decrypt_arm32_crypto_192_out_start_sw: + cmp r4, #2 + blt L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte + ldrh lr, [r9], #2 + sub r4, r4, #2 + strh lr, [r1], #2 +L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte: + cmp r4, #1 + blt L_aes_gcm_decrypt_arm32_crypto_192_out_end_bytes + ldrb lr, [r9], #1 + subs r4, r4, #1 + strb lr, [r1], #1 + bne L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte +L_aes_gcm_decrypt_arm32_crypto_192_out_end_bytes: +L_aes_gcm_decrypt_arm32_crypto_192_partial_done: + # Finish + add r8, r2, #15 + sub r8, r5, r8, lsr #4 + rev r8, r8 + vmov.32 s27, r8 + aese.8 q6, q0 + aesmc.8 q6, q6 + aese.8 q6, q1 + aesmc.8 q6, q6 + aese.8 q6, q2 + aesmc.8 q6, q6 + aese.8 q6, q3 + aesmc.8 q6, q6 + aese.8 q6, q7 + aesmc.8 q6, q6 + aese.8 q6, q8 + aesmc.8 q6, q6 + aese.8 q6, q9 + aesmc.8 q6, q6 + aese.8 q6, q10 + aesmc.8 q6, q6 + aese.8 q6, q11 + aesmc.8 q6, q6 + aese.8 q6, q12 + aesmc.8 q6, q6 + vld1.32 {q14}, [r7]! + aese.8 q6, q13 + aesmc.8 q6, q6 + vld1.32 {q15}, [r7] + aese.8 q6, q14 + veor.8 q6, q6, q15 + sub r7, r7, #16 +#endif /* !NO_AES_192 */ + b L_aes_gcm_decrypt_arm32_crypto_done_enc + # AES_GCM_256 +L_aes_gcm_decrypt_arm32_crypto_start_256: +#ifndef NO_AES_256 + cmp r10, #1 + blt L_aes_gcm_decrypt_arm32_crypto_256_done + beq L_aes_gcm_decrypt_arm32_crypto_256_start_1 +L_aes_gcm_decrypt_arm32_crypto_256_start_2: + add r8, r5, #1 + vmov.8 q4, q6 + add r5, r5, #2 + vmov.8 q5, q6 + rev r8, r8 + rev r12, r5 + vmov s19, r8 + vmov s23, r12 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q5, q0 + aesmc.8 q5, q5 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q5, q1 + aesmc.8 q5, q5 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q5, q2 + aesmc.8 q5, q5 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q5, q3 + aesmc.8 q5, q5 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q5, q7 + aesmc.8 q5, q5 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q5, q8 + aesmc.8 q5, q5 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q5, q9 + aesmc.8 q5, q5 + subs r10, r10, #2 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q5, q10 + aesmc.8 q5, q5 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q5, q11 + aesmc.8 q5, q5 + aese.8 q4, q12 + aesmc.8 q4, q4 + aese.8 q5, q12 + aesmc.8 q5, q5 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + aese.8 q5, q13 + aesmc.8 q5, q5 + vld1.32 {q15}, [r7]! + aese.8 q4, q14 + aesmc.8 q4, q4 + aese.8 q5, q14 + aesmc.8 q5, q5 + vld1.32 {q14}, [r7]! + aese.8 q4, q15 + aesmc.8 q4, q4 + aese.8 q5, q15 + aesmc.8 q5, q5 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + aese.8 q5, q14 + veor.8 q5, q5, q15 + vld1.8 {q14-q15}, [r0]! + sub r7, r7, #48 + veor.8 q14, q14, q4 + veor.8 q15, q15, q5 + vst1.8 {q14-q15}, [r1]! + cmp r10, #1 + bgt L_aes_gcm_decrypt_arm32_crypto_256_start_2 + blt L_aes_gcm_decrypt_arm32_crypto_256_done +L_aes_gcm_decrypt_arm32_crypto_256_start_1: + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q4, q12 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7]! + aese.8 q4, q14 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q15 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + vld1.8 {q14}, [r0]! + sub r7, r7, #48 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r1]! +L_aes_gcm_decrypt_arm32_crypto_256_done: + ands r11, r2, #15 + beq L_aes_gcm_decrypt_arm32_crypto_256_partial_done + veor.8 q14, q14, q14 + mov r4, r11 + vst1.32 {q14}, [r9] + cmp r4, #4 + blt L_aes_gcm_decrypt_arm32_crypto_256_start_sw +L_aes_gcm_decrypt_arm32_crypto_256_start_dw: + ldr lr, [r0], #4 + sub r4, r4, #4 + str lr, [r9], #4 + cmp r4, #4 + bge L_aes_gcm_decrypt_arm32_crypto_256_start_dw +L_aes_gcm_decrypt_arm32_crypto_256_start_sw: + cmp r4, #2 + blt L_aes_gcm_decrypt_arm32_crypto_256_start_byte + ldrh lr, [r0], #2 + sub r4, r4, #2 + strh lr, [r9], #2 +L_aes_gcm_decrypt_arm32_crypto_256_start_byte: + cmp r4, #1 + blt L_aes_gcm_decrypt_arm32_crypto_256_end_bytes + ldrb lr, [r0], #1 + subs r4, r4, #1 + strb lr, [r9], #1 + bne L_aes_gcm_decrypt_arm32_crypto_256_start_byte +L_aes_gcm_decrypt_arm32_crypto_256_end_bytes: + sub r9, r9, r11 + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q4, q12 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q13 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7]! + aese.8 q4, q14 + aesmc.8 q4, q4 + vld1.32 {q14}, [r7]! + aese.8 q4, q15 + aesmc.8 q4, q4 + vld1.32 {q15}, [r7] + aese.8 q4, q14 + veor.8 q4, q4, q15 + vld1.8 {q14}, [r9] + sub r7, r7, #48 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r9] + mov r4, r11 + cmp r4, #4 + blt L_aes_gcm_decrypt_arm32_crypto_256_out_start_sw +L_aes_gcm_decrypt_arm32_crypto_256_out_start_dw: + ldr lr, [r9], #4 + sub r4, r4, #4 + str lr, [r1], #4 + cmp r4, #4 + bge L_aes_gcm_decrypt_arm32_crypto_256_out_start_dw +L_aes_gcm_decrypt_arm32_crypto_256_out_start_sw: + cmp r4, #2 + blt L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte + ldrh lr, [r9], #2 + sub r4, r4, #2 + strh lr, [r1], #2 +L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte: + cmp r4, #1 + blt L_aes_gcm_decrypt_arm32_crypto_256_out_end_bytes + ldrb lr, [r9], #1 + subs r4, r4, #1 + strb lr, [r1], #1 + bne L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte +L_aes_gcm_decrypt_arm32_crypto_256_out_end_bytes: +L_aes_gcm_decrypt_arm32_crypto_256_partial_done: + # Finish + add r8, r2, #15 + sub r8, r5, r8, lsr #4 + rev r8, r8 + vmov.32 s27, r8 + aese.8 q6, q0 + aesmc.8 q6, q6 + aese.8 q6, q1 + aesmc.8 q6, q6 + aese.8 q6, q2 + aesmc.8 q6, q6 + aese.8 q6, q3 + aesmc.8 q6, q6 + aese.8 q6, q7 + aesmc.8 q6, q6 + aese.8 q6, q8 + aesmc.8 q6, q6 + aese.8 q6, q9 + aesmc.8 q6, q6 + aese.8 q6, q10 + aesmc.8 q6, q6 + aese.8 q6, q11 + aesmc.8 q6, q6 + aese.8 q6, q12 + aesmc.8 q6, q6 + vld1.32 {q14}, [r7]! + aese.8 q6, q13 + aesmc.8 q6, q6 + vld1.32 {q15}, [r7]! + aese.8 q6, q14 + aesmc.8 q6, q6 + vld1.32 {q14}, [r7]! + aese.8 q6, q15 + aesmc.8 q6, q6 + vld1.32 {q15}, [r7] + aese.8 q6, q14 + veor.8 q6, q6, q15 + sub r7, r7, #48 +#endif /* !NO_AES_256 */ + b L_aes_gcm_decrypt_arm32_crypto_done_enc + # AES_GCM_128 +L_aes_gcm_decrypt_arm32_crypto_start_128: +#ifndef NO_AES_128 + cmp r10, #1 + blt L_aes_gcm_decrypt_arm32_crypto_128_done + beq L_aes_gcm_decrypt_arm32_crypto_128_start_1 +L_aes_gcm_decrypt_arm32_crypto_128_start_2: + add r8, r5, #1 + vmov.8 q4, q6 + add r5, r5, #2 + vmov.8 q5, q6 + rev r8, r8 + rev r12, r5 + vmov s19, r8 + vmov s23, r12 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q5, q0 + aesmc.8 q5, q5 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q5, q1 + aesmc.8 q5, q5 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q5, q2 + aesmc.8 q5, q5 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q5, q3 + aesmc.8 q5, q5 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q5, q7 + aesmc.8 q5, q5 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q5, q8 + aesmc.8 q5, q5 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q5, q9 + aesmc.8 q5, q5 + subs r10, r10, #2 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q5, q10 + aesmc.8 q5, q5 + aese.8 q4, q11 + aesmc.8 q4, q4 + aese.8 q5, q11 + aesmc.8 q5, q5 + vld1.8 {q14-q15}, [r0]! + aese.8 q4, q12 + veor.8 q4, q4, q13 + aese.8 q5, q12 + veor.8 q5, q5, q13 + veor.8 q14, q14, q4 + veor.8 q15, q15, q5 + vst1.8 {q14-q15}, [r1]! + cmp r10, #1 + bgt L_aes_gcm_decrypt_arm32_crypto_128_start_2 + blt L_aes_gcm_decrypt_arm32_crypto_128_done +L_aes_gcm_decrypt_arm32_crypto_128_start_1: + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + vld1.8 {q14}, [r0]! + aese.8 q4, q12 + veor.8 q4, q4, q13 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r1]! +L_aes_gcm_decrypt_arm32_crypto_128_done: + ands r11, r2, #15 + beq L_aes_gcm_decrypt_arm32_crypto_128_partial_done + veor.8 q14, q14, q14 + mov r4, r11 + vst1.32 {q14}, [r9] + cmp r4, #4 + blt L_aes_gcm_decrypt_arm32_crypto_128_start_sw +L_aes_gcm_decrypt_arm32_crypto_128_start_dw: + ldr lr, [r0], #4 + sub r4, r4, #4 + str lr, [r9], #4 + cmp r4, #4 + bge L_aes_gcm_decrypt_arm32_crypto_128_start_dw +L_aes_gcm_decrypt_arm32_crypto_128_start_sw: + cmp r4, #2 + blt L_aes_gcm_decrypt_arm32_crypto_128_start_byte + ldrh lr, [r0], #2 + sub r4, r4, #2 + strh lr, [r9], #2 +L_aes_gcm_decrypt_arm32_crypto_128_start_byte: + cmp r4, #1 + blt L_aes_gcm_decrypt_arm32_crypto_128_end_bytes + ldrb lr, [r0], #1 + subs r4, r4, #1 + strb lr, [r9], #1 + bne L_aes_gcm_decrypt_arm32_crypto_128_start_byte +L_aes_gcm_decrypt_arm32_crypto_128_end_bytes: + sub r9, r9, r11 + add r5, r5, #1 + vmov.8 q4, q6 + rev r8, r5 + vmov s19, r8 + aese.8 q4, q0 + aesmc.8 q4, q4 + aese.8 q4, q1 + aesmc.8 q4, q4 + aese.8 q4, q2 + aesmc.8 q4, q4 + aese.8 q4, q3 + aesmc.8 q4, q4 + aese.8 q4, q7 + aesmc.8 q4, q4 + aese.8 q4, q8 + aesmc.8 q4, q4 + aese.8 q4, q9 + aesmc.8 q4, q4 + aese.8 q4, q10 + aesmc.8 q4, q4 + aese.8 q4, q11 + aesmc.8 q4, q4 + vld1.8 {q14}, [r9] + aese.8 q4, q12 + veor.8 q4, q4, q13 + veor.8 q14, q14, q4 + vst1.32 {q14}, [r9] + mov r4, r11 + cmp r4, #4 + blt L_aes_gcm_decrypt_arm32_crypto_128_out_start_sw +L_aes_gcm_decrypt_arm32_crypto_128_out_start_dw: + ldr lr, [r9], #4 + sub r4, r4, #4 + str lr, [r1], #4 + cmp r4, #4 + bge L_aes_gcm_decrypt_arm32_crypto_128_out_start_dw +L_aes_gcm_decrypt_arm32_crypto_128_out_start_sw: + cmp r4, #2 + blt L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte + ldrh lr, [r9], #2 + sub r4, r4, #2 + strh lr, [r1], #2 +L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte: + cmp r4, #1 + blt L_aes_gcm_decrypt_arm32_crypto_128_out_end_bytes + ldrb lr, [r9], #1 + subs r4, r4, #1 + strb lr, [r1], #1 + bne L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte +L_aes_gcm_decrypt_arm32_crypto_128_out_end_bytes: +L_aes_gcm_decrypt_arm32_crypto_128_partial_done: + # Finish + add r8, r2, #15 + sub r8, r5, r8, lsr #4 + rev r8, r8 + vmov.32 s27, r8 + aese.8 q6, q0 + aesmc.8 q6, q6 + aese.8 q6, q1 + aesmc.8 q6, q6 + aese.8 q6, q2 + aesmc.8 q6, q6 + aese.8 q6, q3 + aesmc.8 q6, q6 + aese.8 q6, q7 + aesmc.8 q6, q6 + aese.8 q6, q8 + aesmc.8 q6, q6 + aese.8 q6, q9 + aesmc.8 q6, q6 + aese.8 q6, q10 + aesmc.8 q6, q6 + aese.8 q6, q11 + aesmc.8 q6, q6 + aese.8 q6, q12 + veor.8 q6, q6, q13 +#endif /* !NO_AES_128 */ +L_aes_gcm_decrypt_arm32_crypto_done_enc: + vmov.i8 q13, #0x87 + vshr.u64 q13, q13, #56 + # gcm_h + ldr r8, [sp, #124] + vld1.32 {q8}, [r8] + # reg + ldr r9, [sp, #132] + vld1.32 {q7}, [r9] + # tmp + ldr r9, [sp, #128] + lsr lr, r6, #29 + lsl r6, r6, #3 + rbit lr, lr + rbit r6, r6 + vmov s0, lr + vmov s1, r6 + lsr lr, r2, #29 + lsl r2, r2, #3 + rbit lr, lr + rbit r2, r2 + vmov s2, lr + vmov s3, r2 + veor.8 q7, q7, q0 + vmull.p64 q2, d15, d16 + vmull.p64 q3, d14, d17 + vmull.p64 q0, d14, d16 + vmull.p64 q1, d15, d17 + veor.8 q2, q2, q3 + # Reduce + vmull.p64 q7, d3, d27 + veor.8 d2, d2, d5 + veor.8 d1, d1, d4 + veor.8 d2, d2, d15 + veor.8 d1, d1, d14 + vmull.p64 q7, d2, d26 + veor.8 q7, q7, q0 + vmov.i8 q0, #0x55 + vshl.u8 q1, q7, #1 + vshr.u8 q7, q7, #1 + vbif.8 q7, q1, q0 + vmov.i8 q0, #51 + vshl.u8 q1, q7, #2 + vshr.u8 q7, q7, #2 + vbit.8 q1, q7, q0 + vshl.u8 q7, q1, #4 + vsri.u8 q7, q1, #4 + # DONE + veor.8 q7, q7, q6 + # tag + ldr lr, [sp, #104] + # tagSz + ldr r4, [sp, #108] + cmp r4, #16 + blt L_aes_gcm_decrypt_arm32_crypto_tag_part_tag + vld1.8 {q0}, [lr] + b L_aes_gcm_decrypt_arm32_crypto_tag_tag_loaded +L_aes_gcm_decrypt_arm32_crypto_tag_part_tag: + veor.8 q0, q0, q0 + mov r12, r4 + vst1.32 {q0}, [r9] + cmp r12, #4 + blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_sw +L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_dw: + ldr r8, [lr], #4 + sub r12, r12, #4 + str r8, [r9], #4 + cmp r12, #4 + bge L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_dw +L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_sw: + cmp r12, #2 + blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte + ldrh r8, [lr], #2 + sub r12, r12, #2 + strh r8, [r9], #2 +L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte: + cmp r12, #1 + blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_end_bytes + ldrb r8, [lr], #1 + subs r12, r12, #1 + strb r8, [r9], #1 + bne L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte +L_aes_gcm_decrypt_arm32_crypto_tag_tag_end_bytes: + sub r9, r9, r4 + vld1.32 {q0}, [r9] + mov r12, #16 + vst1.32 {q7}, [r9] + sub r12, r12, r4 + eor r8, r8, r8 + add r9, r9, r4 +L_aes_gcm_decrypt_arm32_crypto_tag_calc_tag_byte: + strb r8, [r9], #1 + subs r12, r12, #1 + bne L_aes_gcm_decrypt_arm32_crypto_tag_calc_tag_byte + subs r9, r9, #16 + vld1.32 {q7}, [r9] +L_aes_gcm_decrypt_arm32_crypto_tag_tag_loaded: + vceq.i32 q0, q0, q7 + vmov r5, s0 + vmov r8, s1 + and r12, r5, r8 + vmov r5, s2 + vmov r8, s3 + and r12, r12, r5 + and r12, r12, r8 + mov r5, #-180 + mvn r12, r12 + and r0, r5, r12 + vpop {d8-d15} + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size AES_GCM_decrypt_AARCH32,.-AES_GCM_decrypt_AARCH32 +#endif /* HAVE_AES_DECRYPT */ +#endif /* HAVE_AESGCM */ +#ifdef WOLFSSL_AES_XTS + .text + .align 4 + .globl AES_XTS_encrypt_AARCH32 + .type AES_XTS_encrypt_AARCH32, %function +AES_XTS_encrypt_AARCH32: + push {r4, r5, r6, r7, r8, r9, lr} + vpush {d8-d15} + ldr r12, [sp, #92] + ldr lr, [sp, #96] + ldr r5, [sp, #104] + vldm.32 lr!, {q2-q9} + lsr r4, r2, #4 + and r2, r2, #15 + vld1.8 {q0}, [r3] + cmp r5, #12 + blt L_aes_xts_encrypt_arm32_crypto_start_128 + bgt L_aes_xts_encrypt_arm32_crypto_start_256 + # AES_XTS_192 +#ifndef NO_AES_192 + vldm.32 lr!, {q10-q14} + aese.8 q0, q2 + aesmc.8 q0, q0 + aese.8 q0, q3 + aesmc.8 q0, q0 + aese.8 q0, q4 + aesmc.8 q0, q0 + aese.8 q0, q5 + aesmc.8 q0, q0 + aese.8 q0, q6 + aesmc.8 q0, q0 + aese.8 q0, q7 + aesmc.8 q0, q0 + aese.8 q0, q8 + aesmc.8 q0, q0 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q0, q11 + aesmc.8 q0, q0 + aese.8 q0, q12 + aesmc.8 q0, q0 + aese.8 q0, q13 + veor.32 q0, q0, q14 + vmov r6, r7, d0 + vmov r8, r9, d1 + vldm.32 r12!, {q2-q9} + vldm.32 r12!, {q10-q14} + mov lr, #0x87 + cmp r4, #1 + blt L_aes_xts_encrypt_arm32_crypto_192_done +L_aes_xts_encrypt_arm32_crypto_192_start_1: + vld1.8 {q1}, [r0]! + veor.32 q1, q1, q0 + aese.8 q1, q2 + aesmc.8 q1, q1 + and r5, lr, r9, asr #31 + aese.8 q1, q3 + aesmc.8 q1, q1 + lsl r9, r9, #1 + aese.8 q1, q4 + aesmc.8 q1, q1 + orr r9, r9, r8, lsr #31 + aese.8 q1, q5 + aesmc.8 q1, q1 + lsl r8, r8, #1 + aese.8 q1, q6 + aesmc.8 q1, q1 + orr r8, r8, r7, lsr #31 + aese.8 q1, q7 + aesmc.8 q1, q1 + lsl r7, r7, #1 + aese.8 q1, q8 + aesmc.8 q1, q1 + orr r7, r7, r6, lsr #31 + aese.8 q1, q9 + aesmc.8 q1, q1 + eor r6, r5, r6, lsl #1 + aese.8 q1, q10 + aesmc.8 q1, q1 + aese.8 q1, q11 + aesmc.8 q1, q1 + aese.8 q1, q12 + aesmc.8 q1, q1 + aese.8 q1, q13 + veor.32 q1, q1, q14 + veor.32 q1, q1, q0 + vmov d0, r6, r7 + vmov d1, r8, r9 + subs r4, r4, #1 + vst1.8 {q1}, [r1]! + bne L_aes_xts_encrypt_arm32_crypto_192_start_1 +L_aes_xts_encrypt_arm32_crypto_192_done: + cmp r2, #0 + beq L_aes_xts_encrypt_arm32_crypto_192_partial_done + sub r1, r1, #16 + ldr r4, [sp, #100] + vld1.8 {q1}, [r1]! + vst1.32 {q1}, [r4] + mov r5, r2 +L_aes_xts_encrypt_arm32_crypto_192_start_byte: + ldrb r8, [r4] + ldrb r9, [r0], #1 + strb r8, [r1], #1 + strb r9, [r4], #1 + subs r5, r5, #1 + bgt L_aes_xts_encrypt_arm32_crypto_192_start_byte + sub r1, r1, r2 + sub r4, r4, r2 + sub r1, r1, #16 + vld1.32 {q1}, [r4] + veor.32 q1, q1, q0 + aese.8 q1, q2 + aesmc.8 q1, q1 + aese.8 q1, q3 + aesmc.8 q1, q1 + aese.8 q1, q4 + aesmc.8 q1, q1 + aese.8 q1, q5 + aesmc.8 q1, q1 + aese.8 q1, q6 + aesmc.8 q1, q1 + aese.8 q1, q7 + aesmc.8 q1, q1 + aese.8 q1, q8 + aesmc.8 q1, q1 + aese.8 q1, q9 + aesmc.8 q1, q1 + aese.8 q1, q10 + aesmc.8 q1, q1 + aese.8 q1, q11 + aesmc.8 q1, q1 + aese.8 q1, q12 + aesmc.8 q1, q1 + aese.8 q1, q13 + veor.32 q1, q1, q14 + veor.32 q1, q1, q0 + vst1.8 {q1}, [r1] +L_aes_xts_encrypt_arm32_crypto_192_partial_done: +#endif /* !NO_AES_192 */ + b L_aes_xts_encrypt_arm32_crypto_done + # AES_XTS_256 +L_aes_xts_encrypt_arm32_crypto_start_256: +#ifndef NO_AES_256 + vldm.32 lr!, {q10-q13} + aese.8 q0, q2 + aesmc.8 q0, q0 + aese.8 q0, q3 + aesmc.8 q0, q0 + aese.8 q0, q4 + aesmc.8 q0, q0 + aese.8 q0, q5 + aesmc.8 q0, q0 + aese.8 q0, q6 + aesmc.8 q0, q0 + aese.8 q0, q7 + aesmc.8 q0, q0 + aese.8 q0, q8 + aesmc.8 q0, q0 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q0, q11 + aesmc.8 q0, q0 + aese.8 q0, q12 + aesmc.8 q0, q0 + vld1.32 {q14}, [lr]! + aese.8 q0, q13 + aesmc.8 q0, q0 + vld1.32 {q15}, [lr]! + aese.8 q0, q14 + aesmc.8 q0, q0 + vld1.32 {q14}, [lr]! + aese.8 q0, q15 + veor.32 q0, q0, q14 + vmov r6, r7, d0 + vmov r8, r9, d1 + vldm.32 r12!, {q2-q9} + vldm.32 r12!, {q10-q13} + mov lr, #0x87 + cmp r4, #1 + blt L_aes_xts_encrypt_arm32_crypto_256_done +L_aes_xts_encrypt_arm32_crypto_256_start_1: + vld1.8 {q1}, [r0]! + veor.32 q1, q1, q0 + aese.8 q1, q2 + aesmc.8 q1, q1 + and r5, lr, r9, asr #31 + aese.8 q1, q3 + aesmc.8 q1, q1 + lsl r9, r9, #1 + aese.8 q1, q4 + aesmc.8 q1, q1 + orr r9, r9, r8, lsr #31 + aese.8 q1, q5 + aesmc.8 q1, q1 + lsl r8, r8, #1 + aese.8 q1, q6 + aesmc.8 q1, q1 + orr r8, r8, r7, lsr #31 + aese.8 q1, q7 + aesmc.8 q1, q1 + lsl r7, r7, #1 + aese.8 q1, q8 + aesmc.8 q1, q1 + orr r7, r7, r6, lsr #31 + aese.8 q1, q9 + aesmc.8 q1, q1 + eor r6, r5, r6, lsl #1 + aese.8 q1, q10 + aesmc.8 q1, q1 + aese.8 q1, q11 + aesmc.8 q1, q1 + aese.8 q1, q12 + aesmc.8 q1, q1 + vld1.32 {q14}, [r12]! + aese.8 q1, q13 + aesmc.8 q1, q1 + vld1.32 {q15}, [r12]! + aese.8 q1, q14 + aesmc.8 q1, q1 + vld1.32 {q14}, [r12]! + aese.8 q1, q15 + veor.32 q1, q1, q14 + sub r12, r12, #48 + veor.32 q1, q1, q0 + vmov d0, r6, r7 + vmov d1, r8, r9 + subs r4, r4, #1 + vst1.8 {q1}, [r1]! + bne L_aes_xts_encrypt_arm32_crypto_256_start_1 +L_aes_xts_encrypt_arm32_crypto_256_done: + cmp r2, #0 + beq L_aes_xts_encrypt_arm32_crypto_256_partial_done + sub r1, r1, #16 + ldr r4, [sp, #100] + vld1.8 {q1}, [r1]! + vst1.32 {q1}, [r4] + mov r5, r2 +L_aes_xts_encrypt_arm32_crypto_256_start_byte: + ldrb r8, [r4] + ldrb r9, [r0], #1 + strb r8, [r1], #1 + strb r9, [r4], #1 + subs r5, r5, #1 + bgt L_aes_xts_encrypt_arm32_crypto_256_start_byte + sub r1, r1, r2 + sub r4, r4, r2 + sub r1, r1, #16 + vld1.32 {q1}, [r4] + veor.32 q1, q1, q0 + aese.8 q1, q2 + aesmc.8 q1, q1 + aese.8 q1, q3 + aesmc.8 q1, q1 + aese.8 q1, q4 + aesmc.8 q1, q1 + aese.8 q1, q5 + aesmc.8 q1, q1 + aese.8 q1, q6 + aesmc.8 q1, q1 + aese.8 q1, q7 + aesmc.8 q1, q1 + aese.8 q1, q8 + aesmc.8 q1, q1 + aese.8 q1, q9 + aesmc.8 q1, q1 + aese.8 q1, q10 + aesmc.8 q1, q1 + aese.8 q1, q11 + aesmc.8 q1, q1 + aese.8 q1, q12 + aesmc.8 q1, q1 + vld1.32 {q14}, [r12]! + aese.8 q1, q13 + aesmc.8 q1, q1 + vld1.32 {q15}, [r12]! + aese.8 q1, q14 + aesmc.8 q1, q1 + vld1.32 {q14}, [r12]! + aese.8 q1, q15 + veor.32 q1, q1, q14 + veor.32 q1, q1, q0 + vst1.8 {q1}, [r1] +L_aes_xts_encrypt_arm32_crypto_256_partial_done: +#endif /* !NO_AES_256 */ + b L_aes_xts_encrypt_arm32_crypto_done + # AES_XTS_128 +L_aes_xts_encrypt_arm32_crypto_start_128: +#ifndef NO_AES_128 + vldm.32 lr!, {q10-q12} + aese.8 q0, q2 + aesmc.8 q0, q0 + aese.8 q0, q3 + aesmc.8 q0, q0 + aese.8 q0, q4 + aesmc.8 q0, q0 + aese.8 q0, q5 + aesmc.8 q0, q0 + aese.8 q0, q6 + aesmc.8 q0, q0 + aese.8 q0, q7 + aesmc.8 q0, q0 + aese.8 q0, q8 + aesmc.8 q0, q0 + aese.8 q0, q9 + aesmc.8 q0, q0 + aese.8 q0, q10 + aesmc.8 q0, q0 + aese.8 q0, q11 + veor.32 q0, q0, q12 + vmov r6, r7, d0 + vmov r8, r9, d1 + vldm.32 r12!, {q2-q9} + vldm.32 r12!, {q10-q12} + mov lr, #0x87 + cmp r4, #1 + blt L_aes_xts_encrypt_arm32_crypto_128_done +L_aes_xts_encrypt_arm32_crypto_128_start_1: + vld1.8 {q1}, [r0]! + veor.32 q1, q1, q0 + aese.8 q1, q2 + aesmc.8 q1, q1 + and r5, lr, r9, asr #31 + aese.8 q1, q3 + aesmc.8 q1, q1 + lsl r9, r9, #1 + aese.8 q1, q4 + aesmc.8 q1, q1 + orr r9, r9, r8, lsr #31 + aese.8 q1, q5 + aesmc.8 q1, q1 + lsl r8, r8, #1 + aese.8 q1, q6 + aesmc.8 q1, q1 + orr r8, r8, r7, lsr #31 + aese.8 q1, q7 + aesmc.8 q1, q1 + lsl r7, r7, #1 + aese.8 q1, q8 + aesmc.8 q1, q1 + orr r7, r7, r6, lsr #31 + aese.8 q1, q9 + aesmc.8 q1, q1 + eor r6, r5, r6, lsl #1 + aese.8 q1, q10 + aesmc.8 q1, q1 + aese.8 q1, q11 + veor.32 q1, q1, q12 + veor.32 q1, q1, q0 + vmov d0, r6, r7 + vmov d1, r8, r9 + subs r4, r4, #1 + vst1.8 {q1}, [r1]! + bne L_aes_xts_encrypt_arm32_crypto_128_start_1 +L_aes_xts_encrypt_arm32_crypto_128_done: + cmp r2, #0 + beq L_aes_xts_encrypt_arm32_crypto_128_partial_done + sub r1, r1, #16 + ldr r4, [sp, #100] + vld1.8 {q1}, [r1]! + vst1.32 {q1}, [r4] + mov r5, r2 +L_aes_xts_encrypt_arm32_crypto_128_start_byte: + ldrb r8, [r4] + ldrb r9, [r0], #1 + strb r8, [r1], #1 + strb r9, [r4], #1 + subs r5, r5, #1 + bgt L_aes_xts_encrypt_arm32_crypto_128_start_byte + sub r1, r1, r2 + sub r4, r4, r2 + sub r1, r1, #16 + vld1.32 {q1}, [r4] + veor.32 q1, q1, q0 + aese.8 q1, q2 + aesmc.8 q1, q1 + aese.8 q1, q3 + aesmc.8 q1, q1 + aese.8 q1, q4 + aesmc.8 q1, q1 + aese.8 q1, q5 + aesmc.8 q1, q1 + aese.8 q1, q6 + aesmc.8 q1, q1 + aese.8 q1, q7 + aesmc.8 q1, q1 + aese.8 q1, q8 + aesmc.8 q1, q1 + aese.8 q1, q9 + aesmc.8 q1, q1 + aese.8 q1, q10 + aesmc.8 q1, q1 + aese.8 q1, q11 + veor.32 q1, q1, q12 + veor.32 q1, q1, q0 + vst1.8 {q1}, [r1] +L_aes_xts_encrypt_arm32_crypto_128_partial_done: +#endif /* !NO_AES_128 */ +L_aes_xts_encrypt_arm32_crypto_done: + vpop {d8-d15} + pop {r4, r5, r6, r7, r8, r9, pc} + .size AES_XTS_encrypt_AARCH32,.-AES_XTS_encrypt_AARCH32 +#ifdef HAVE_AES_DECRYPT + .text + .align 4 + .globl AES_XTS_decrypt_AARCH32 + .type AES_XTS_decrypt_AARCH32, %function +AES_XTS_decrypt_AARCH32: + push {r4, r5, r6, r7, r8, r9, lr} + vpush {d8-d15} + ldr r12, [sp, #92] + ldr lr, [sp, #96] + vldm.32 lr!, {q2-q9} + eor r5, r5, r5 + lsr r4, r2, #4 + ands r2, r2, #15 + sub r5, r5, r2 + sub r4, r4, r5, lsr #31 + vld1.8 {q1}, [r3] + ldr r5, [sp, #104] + cmp r5, #12 + blt L_aes_xts_decrypt_arm32_crypto_start_128 + bgt L_aes_xts_decrypt_arm32_crypto_start_256 + # AES_XTS_192 +#ifndef NO_AES_192 + vldm.32 lr!, {q10-q14} + aese.8 q1, q2 + aesmc.8 q1, q1 + aese.8 q1, q3 + aesmc.8 q1, q1 + aese.8 q1, q4 + aesmc.8 q1, q1 + aese.8 q1, q5 + aesmc.8 q1, q1 + aese.8 q1, q6 + aesmc.8 q1, q1 + aese.8 q1, q7 + aesmc.8 q1, q1 + aese.8 q1, q8 + aesmc.8 q1, q1 + aese.8 q1, q9 + aesmc.8 q1, q1 + aese.8 q1, q10 + aesmc.8 q1, q1 + aese.8 q1, q11 + aesmc.8 q1, q1 + aese.8 q1, q12 + aesmc.8 q1, q1 + aese.8 q1, q13 + veor.32 q1, q1, q14 + vmov r6, r7, d2 + vmov r8, r9, d3 + vldm.32 r12!, {q2-q9} + vldm.32 r12!, {q10-q14} + mov lr, #0x87 + cmp r4, #1 + blt L_aes_xts_decrypt_arm32_crypto_192_done +L_aes_xts_decrypt_arm32_crypto_192_start_1: + vld1.8 {q0}, [r0]! + veor.32 q0, q0, q1 + aesd.8 q0, q2 + aesimc.8 q0, q0 + and r5, lr, r9, asr #31 + aesd.8 q0, q3 + aesimc.8 q0, q0 + lsl r9, r9, #1 + aesd.8 q0, q4 + aesimc.8 q0, q0 + orr r9, r9, r8, lsr #31 + aesd.8 q0, q5 + aesimc.8 q0, q0 + lsl r8, r8, #1 + aesd.8 q0, q6 + aesimc.8 q0, q0 + orr r8, r8, r7, lsr #31 + aesd.8 q0, q7 + aesimc.8 q0, q0 + lsl r7, r7, #1 + aesd.8 q0, q8 + aesimc.8 q0, q0 + orr r7, r7, r6, lsr #31 + aesd.8 q0, q9 + aesimc.8 q0, q0 + eor r6, r5, r6, lsl #1 + aesd.8 q0, q10 + aesimc.8 q0, q0 + aesd.8 q0, q11 + aesimc.8 q0, q0 + aesd.8 q0, q12 + aesimc.8 q0, q0 + aesd.8 q0, q13 + veor.32 q0, q0, q14 + veor.32 q0, q0, q1 + vmov d2, r6, r7 + vmov d3, r8, r9 + subs r4, r4, #1 + vst1.8 {q0}, [r1]! + bne L_aes_xts_decrypt_arm32_crypto_192_start_1 +L_aes_xts_decrypt_arm32_crypto_192_done: + cmp r2, #0 + beq L_aes_xts_decrypt_arm32_crypto_192_partial_done + and r5, lr, r9, asr #31 + lsl r9, r9, #1 + orr r9, r9, r8, lsr #31 + lsl r8, r8, #1 + orr r8, r8, r7, lsr #31 + lsl r7, r7, #1 + orr r7, r7, r6, lsr #31 + eor r6, r5, r6, lsl #1 + vmov d30, r6, r7 + vmov d31, r8, r9 + vld1.8 {q0}, [r0]! + ldr r4, [sp, #100] + veor.32 q0, q0, q15 + aesd.8 q0, q2 + aesimc.8 q0, q0 + aesd.8 q0, q3 + aesimc.8 q0, q0 + aesd.8 q0, q4 + aesimc.8 q0, q0 + aesd.8 q0, q5 + aesimc.8 q0, q0 + aesd.8 q0, q6 + aesimc.8 q0, q0 + aesd.8 q0, q7 + aesimc.8 q0, q0 + aesd.8 q0, q8 + aesimc.8 q0, q0 + aesd.8 q0, q9 + aesimc.8 q0, q0 + aesd.8 q0, q10 + aesimc.8 q0, q0 + aesd.8 q0, q11 + aesimc.8 q0, q0 + aesd.8 q0, q12 + aesimc.8 q0, q0 + aesd.8 q0, q13 + veor.32 q0, q0, q14 + veor.32 q0, q0, q15 + vst1.32 {q0}, [r4] + add r1, r1, #16 + mov r5, r2 +L_aes_xts_decrypt_arm32_crypto_192_start_byte: + ldrb r8, [r4] + ldrb r9, [r0], #1 + strb r8, [r1], #1 + strb r9, [r4], #1 + subs r5, r5, #1 + bgt L_aes_xts_decrypt_arm32_crypto_192_start_byte + sub r1, r1, r2 + sub r4, r4, r2 + sub r1, r1, #16 + vld1.32 {q0}, [r4] + veor.32 q0, q0, q1 + aesd.8 q0, q2 + aesimc.8 q0, q0 + aesd.8 q0, q3 + aesimc.8 q0, q0 + aesd.8 q0, q4 + aesimc.8 q0, q0 + aesd.8 q0, q5 + aesimc.8 q0, q0 + aesd.8 q0, q6 + aesimc.8 q0, q0 + aesd.8 q0, q7 + aesimc.8 q0, q0 + aesd.8 q0, q8 + aesimc.8 q0, q0 + aesd.8 q0, q9 + aesimc.8 q0, q0 + aesd.8 q0, q10 + aesimc.8 q0, q0 + aesd.8 q0, q11 + aesimc.8 q0, q0 + aesd.8 q0, q12 + aesimc.8 q0, q0 + aesd.8 q0, q13 + veor.32 q0, q0, q14 + veor.32 q0, q0, q1 + vst1.8 {q0}, [r1] +L_aes_xts_decrypt_arm32_crypto_192_partial_done: +#endif /* !NO_AES_192 */ + b L_aes_xts_decrypt_arm32_crypto_done + # AES_XTS_256 +L_aes_xts_decrypt_arm32_crypto_start_256: +#ifndef NO_AES_256 + vldm.32 lr!, {q10-q13} + aese.8 q1, q2 + aesmc.8 q1, q1 + aese.8 q1, q3 + aesmc.8 q1, q1 + aese.8 q1, q4 + aesmc.8 q1, q1 + aese.8 q1, q5 + aesmc.8 q1, q1 + aese.8 q1, q6 + aesmc.8 q1, q1 + aese.8 q1, q7 + aesmc.8 q1, q1 + aese.8 q1, q8 + aesmc.8 q1, q1 + aese.8 q1, q9 + aesmc.8 q1, q1 + aese.8 q1, q10 + aesmc.8 q1, q1 + aese.8 q1, q11 + aesmc.8 q1, q1 + aese.8 q1, q12 + aesmc.8 q1, q1 + vld1.32 {q14}, [lr]! + aese.8 q1, q13 + aesmc.8 q1, q1 + vld1.32 {q15}, [lr]! + aese.8 q1, q14 + aesmc.8 q1, q1 + vld1.32 {q14}, [lr]! + aese.8 q1, q15 + veor.32 q1, q1, q14 + vmov r6, r7, d2 + vmov r8, r9, d3 + vldm.32 r12!, {q2-q9} + vldm.32 r12!, {q10-q12} + mov lr, #0x87 + cmp r4, #1 + blt L_aes_xts_decrypt_arm32_crypto_256_done +L_aes_xts_decrypt_arm32_crypto_256_start_1: + vld1.8 {q0}, [r0]! + veor.32 q0, q0, q1 + aesd.8 q0, q2 + aesimc.8 q0, q0 + and r5, lr, r9, asr #31 + aesd.8 q0, q3 + aesimc.8 q0, q0 + lsl r9, r9, #1 + aesd.8 q0, q4 + aesimc.8 q0, q0 + orr r9, r9, r8, lsr #31 + aesd.8 q0, q5 + aesimc.8 q0, q0 + lsl r8, r8, #1 + aesd.8 q0, q6 + aesimc.8 q0, q0 + orr r8, r8, r7, lsr #31 + aesd.8 q0, q7 + aesimc.8 q0, q0 + lsl r7, r7, #1 + aesd.8 q0, q8 + aesimc.8 q0, q0 + orr r7, r7, r6, lsr #31 + aesd.8 q0, q9 + aesimc.8 q0, q0 + eor r6, r5, r6, lsl #1 + aesd.8 q0, q10 + aesimc.8 q0, q0 + aesd.8 q0, q11 + aesimc.8 q0, q0 + vld1.32 {q13}, [r12]! + aesd.8 q0, q12 + aesimc.8 q0, q0 + vld1.32 {q14}, [r12]! + aesd.8 q0, q13 + aesimc.8 q0, q0 + vld1.32 {q13}, [r12]! + aesd.8 q0, q14 + aesimc.8 q0, q0 + vld1.32 {q14}, [r12] + aesd.8 q0, q13 + veor.32 q0, q0, q14 + sub r12, r12, #48 + veor.32 q0, q0, q1 + vmov d2, r6, r7 + vmov d3, r8, r9 + subs r4, r4, #1 + vst1.8 {q0}, [r1]! + bne L_aes_xts_decrypt_arm32_crypto_256_start_1 +L_aes_xts_decrypt_arm32_crypto_256_done: + cmp r2, #0 + beq L_aes_xts_decrypt_arm32_crypto_256_partial_done + and r5, lr, r9, asr #31 + lsl r9, r9, #1 + orr r9, r9, r8, lsr #31 + lsl r8, r8, #1 + orr r8, r8, r7, lsr #31 + lsl r7, r7, #1 + orr r7, r7, r6, lsr #31 + eor r6, r5, r6, lsl #1 + vmov d30, r6, r7 + vmov d31, r8, r9 + vld1.8 {q0}, [r0]! + ldr r4, [sp, #100] + veor.32 q0, q0, q15 + aesd.8 q0, q2 + aesimc.8 q0, q0 + aesd.8 q0, q3 + aesimc.8 q0, q0 + aesd.8 q0, q4 + aesimc.8 q0, q0 + aesd.8 q0, q5 + aesimc.8 q0, q0 + aesd.8 q0, q6 + aesimc.8 q0, q0 + aesd.8 q0, q7 + aesimc.8 q0, q0 + aesd.8 q0, q8 + aesimc.8 q0, q0 + aesd.8 q0, q9 + aesimc.8 q0, q0 + aesd.8 q0, q10 + aesimc.8 q0, q0 + aesd.8 q0, q11 + aesimc.8 q0, q0 + vld1.32 {q13}, [r12]! + aesd.8 q0, q12 + aesimc.8 q0, q0 + vld1.32 {q14}, [r12]! + aesd.8 q0, q13 + aesimc.8 q0, q0 + vld1.32 {q13}, [r12]! + aesd.8 q0, q14 + aesimc.8 q0, q0 + vld1.32 {q14}, [r12] + aesd.8 q0, q13 + veor.32 q0, q0, q14 + sub r12, r12, #48 + veor.32 q0, q0, q15 + vst1.32 {q0}, [r4] + add r1, r1, #16 + mov r5, r2 +L_aes_xts_decrypt_arm32_crypto_256_start_byte: + ldrb r8, [r4] + ldrb r9, [r0], #1 + strb r8, [r1], #1 + strb r9, [r4], #1 + subs r5, r5, #1 + bgt L_aes_xts_decrypt_arm32_crypto_256_start_byte + sub r1, r1, r2 + sub r4, r4, r2 + sub r1, r1, #16 + vld1.32 {q0}, [r4] + veor.32 q0, q0, q1 + aesd.8 q0, q2 + aesimc.8 q0, q0 + aesd.8 q0, q3 + aesimc.8 q0, q0 + aesd.8 q0, q4 + aesimc.8 q0, q0 + aesd.8 q0, q5 + aesimc.8 q0, q0 + aesd.8 q0, q6 + aesimc.8 q0, q0 + aesd.8 q0, q7 + aesimc.8 q0, q0 + aesd.8 q0, q8 + aesimc.8 q0, q0 + aesd.8 q0, q9 + aesimc.8 q0, q0 + aesd.8 q0, q10 + aesimc.8 q0, q0 + aesd.8 q0, q11 + aesimc.8 q0, q0 + vld1.32 {q13}, [r12]! + aesd.8 q0, q12 + aesimc.8 q0, q0 + vld1.32 {q14}, [r12]! + aesd.8 q0, q13 + aesimc.8 q0, q0 + vld1.32 {q13}, [r12]! + aesd.8 q0, q14 + aesimc.8 q0, q0 + vld1.32 {q14}, [r12] + aesd.8 q0, q13 + veor.32 q0, q0, q14 + veor.32 q0, q0, q1 + vst1.8 {q0}, [r1] +L_aes_xts_decrypt_arm32_crypto_256_partial_done: +#endif /* !NO_AES_256 */ + b L_aes_xts_decrypt_arm32_crypto_done + # AES_XTS_128 +L_aes_xts_decrypt_arm32_crypto_start_128: +#ifndef NO_AES_128 + vldm.32 lr!, {q10-q12} + aese.8 q1, q2 + aesmc.8 q1, q1 + aese.8 q1, q3 + aesmc.8 q1, q1 + aese.8 q1, q4 + aesmc.8 q1, q1 + aese.8 q1, q5 + aesmc.8 q1, q1 + aese.8 q1, q6 + aesmc.8 q1, q1 + aese.8 q1, q7 + aesmc.8 q1, q1 + aese.8 q1, q8 + aesmc.8 q1, q1 + aese.8 q1, q9 + aesmc.8 q1, q1 + aese.8 q1, q10 + aesmc.8 q1, q1 + aese.8 q1, q11 + veor.32 q1, q1, q12 + vmov r6, r7, d2 + vmov r8, r9, d3 + vldm.32 r12!, {q2-q9} + vldm.32 r12!, {q10-q12} + mov lr, #0x87 + cmp r4, #1 + blt L_aes_xts_decrypt_arm32_crypto_128_done +L_aes_xts_decrypt_arm32_crypto_128_start_1: + vld1.8 {q0}, [r0]! + veor.32 q0, q0, q1 + aesd.8 q0, q2 + aesimc.8 q0, q0 + and r5, lr, r9, asr #31 + aesd.8 q0, q3 + aesimc.8 q0, q0 + lsl r9, r9, #1 + aesd.8 q0, q4 + aesimc.8 q0, q0 + orr r9, r9, r8, lsr #31 + aesd.8 q0, q5 + aesimc.8 q0, q0 + lsl r8, r8, #1 + aesd.8 q0, q6 + aesimc.8 q0, q0 + orr r8, r8, r7, lsr #31 + aesd.8 q0, q7 + aesimc.8 q0, q0 + lsl r7, r7, #1 + aesd.8 q0, q8 + aesimc.8 q0, q0 + orr r7, r7, r6, lsr #31 + aesd.8 q0, q9 + aesimc.8 q0, q0 + eor r6, r5, r6, lsl #1 + aesd.8 q0, q10 + aesimc.8 q0, q0 + aesd.8 q0, q11 + veor.32 q0, q0, q12 + veor.32 q0, q0, q1 + vmov d2, r6, r7 + vmov d3, r8, r9 + subs r4, r4, #1 + vst1.8 {q0}, [r1]! + bne L_aes_xts_decrypt_arm32_crypto_128_start_1 +L_aes_xts_decrypt_arm32_crypto_128_done: + cmp r2, #0 + beq L_aes_xts_decrypt_arm32_crypto_128_partial_done + and r5, lr, r9, asr #31 + lsl r9, r9, #1 + orr r9, r9, r8, lsr #31 + lsl r8, r8, #1 + orr r8, r8, r7, lsr #31 + lsl r7, r7, #1 + orr r7, r7, r6, lsr #31 + eor r6, r5, r6, lsl #1 + vmov d30, r6, r7 + vmov d31, r8, r9 + vld1.8 {q0}, [r0]! + ldr r4, [sp, #100] + veor.32 q0, q0, q15 + aesd.8 q0, q2 + aesimc.8 q0, q0 + aesd.8 q0, q3 + aesimc.8 q0, q0 + aesd.8 q0, q4 + aesimc.8 q0, q0 + aesd.8 q0, q5 + aesimc.8 q0, q0 + aesd.8 q0, q6 + aesimc.8 q0, q0 + aesd.8 q0, q7 + aesimc.8 q0, q0 + aesd.8 q0, q8 + aesimc.8 q0, q0 + aesd.8 q0, q9 + aesimc.8 q0, q0 + aesd.8 q0, q10 + aesimc.8 q0, q0 + aesd.8 q0, q11 + veor.32 q0, q0, q12 + veor.32 q0, q0, q15 + vst1.32 {q0}, [r4] add r1, r1, #16 - bne L_AES_GCMSIV_ctr_base_loop_block_128 -L_AES_GCMSIV_ctr_base_end: - pop {r3, r8} -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - eor r10, r4, r4, ror #16 - eor r11, r5, r5, ror #16 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r4, r4, #8 - ror r5, r5, #8 - eor r4, r4, r10, lsr #8 - eor r5, r5, r11, lsr #8 - eor r10, r6, r6, ror #16 - eor r11, r7, r7, ror #16 - bic r10, r10, #0xff0000 - bic r11, r11, #0xff0000 - ror r6, r6, #8 - ror r7, r7, #8 - eor r6, r6, r10, lsr #8 - eor r7, r7, r11, lsr #8 -#else - rev r4, r4 - rev r5, r5 - rev r6, r6 - rev r7, r7 -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - stm r8, {r4, r5, r6, r7} - pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} - .size AES_GCMSIV_ctr_base,.-AES_GCMSIV_ctr_base + mov r5, r2 +L_aes_xts_decrypt_arm32_crypto_128_start_byte: + ldrb r8, [r4] + ldrb r9, [r0], #1 + strb r8, [r1], #1 + strb r9, [r4], #1 + subs r5, r5, #1 + bgt L_aes_xts_decrypt_arm32_crypto_128_start_byte + sub r1, r1, r2 + sub r4, r4, r2 + sub r1, r1, #16 + vld1.32 {q0}, [r4] + veor.32 q0, q0, q1 + aesd.8 q0, q2 + aesimc.8 q0, q0 + aesd.8 q0, q3 + aesimc.8 q0, q0 + aesd.8 q0, q4 + aesimc.8 q0, q0 + aesd.8 q0, q5 + aesimc.8 q0, q0 + aesd.8 q0, q6 + aesimc.8 q0, q0 + aesd.8 q0, q7 + aesimc.8 q0, q0 + aesd.8 q0, q8 + aesimc.8 q0, q0 + aesd.8 q0, q9 + aesimc.8 q0, q0 + aesd.8 q0, q10 + aesimc.8 q0, q0 + aesd.8 q0, q11 + veor.32 q0, q0, q12 + veor.32 q0, q0, q1 + vst1.8 {q0}, [r1] +L_aes_xts_decrypt_arm32_crypto_128_partial_done: +#endif /* !NO_AES_128 */ +L_aes_xts_decrypt_arm32_crypto_done: + vpop {d8-d15} + pop {r4, r5, r6, r7, r8, r9, pc} + .size AES_XTS_decrypt_AARCH32,.-AES_XTS_decrypt_AARCH32 +#endif /* HAVE_AES_DECRYPT */ +#endif /* WOLFSSL_AES_XTS */ +#ifdef WOLFSSL_AESGCM_SIV + .text + .align 4 + .globl AES_GCMSIV_polyval_crypto + .type AES_GCMSIV_polyval_crypto, %function +AES_GCMSIV_polyval_crypto: + vpush {d8-d15} + veor.8 q2, q2, q2 + vld1.8 {q1}, [r1] + vld1.8 {q0}, [r0] + vrev64.8 q0, q0 + vext.8 q0, q0, q0, #8 + vmull.p64 q7, d2, d2 + vmull.p64 q11, d2, d3 + vmull.p64 q12, d3, d2 + vmull.p64 q8, d3, d3 + veor.8 q11, q11, q12 + vext.8 q12, q2, q11, #8 + vext.8 q11, q11, q2, #8 + veor.8 q7, q7, q12 + veor.8 q8, q8, q11 + vshr.u32 q6, q7, #31 + vshr.u32 q9, q8, #31 + vshl.i32 q7, q7, #1 + vshl.i32 q8, q8, #1 + vext.8 q10, q6, q2, #12 + vext.8 q9, q2, q9, #12 + vext.8 q6, q2, q6, #12 + veor.8 q7, q7, q6 + veor.8 q8, q8, q9 + veor.8 q8, q8, q10 + vshl.i32 q6, q7, #31 + vshl.i32 q9, q7, #30 + vshl.i32 q10, q7, #25 + veor.8 q6, q6, q9 + veor.8 q6, q6, q10 + vext.8 q9, q6, q2, #4 + vext.8 q6, q2, q6, #4 + veor.8 q7, q7, q6 + vshr.u32 q11, q7, #1 + vshr.u32 q12, q7, #2 + vshr.u32 q13, q7, #7 + veor.8 q11, q11, q12 + veor.8 q11, q11, q13 + veor.8 q11, q11, q9 + veor.8 q7, q7, q11 + veor.8 q3, q8, q7 + vmull.p64 q7, d6, d2 + vmull.p64 q11, d6, d3 + vmull.p64 q12, d7, d2 + vmull.p64 q8, d7, d3 + veor.8 q11, q11, q12 + vext.8 q12, q2, q11, #8 + vext.8 q11, q11, q2, #8 + veor.8 q7, q7, q12 + veor.8 q8, q8, q11 + vshr.u32 q6, q7, #31 + vshr.u32 q9, q8, #31 + vshl.i32 q7, q7, #1 + vshl.i32 q8, q8, #1 + vext.8 q10, q6, q2, #12 + vext.8 q9, q2, q9, #12 + vext.8 q6, q2, q6, #12 + veor.8 q7, q7, q6 + veor.8 q8, q8, q9 + veor.8 q8, q8, q10 + vshl.i32 q6, q7, #31 + vshl.i32 q9, q7, #30 + vshl.i32 q10, q7, #25 + veor.8 q6, q6, q9 + veor.8 q6, q6, q10 + vext.8 q9, q6, q2, #4 + vext.8 q6, q2, q6, #4 + veor.8 q7, q7, q6 + vshr.u32 q11, q7, #1 + vshr.u32 q12, q7, #2 + vshr.u32 q13, q7, #7 + veor.8 q11, q11, q12 + veor.8 q11, q11, q13 + veor.8 q11, q11, q9 + veor.8 q7, q7, q11 + veor.8 q4, q8, q7 + vmull.p64 q7, d6, d6 + vmull.p64 q11, d6, d7 + vmull.p64 q12, d7, d6 + vmull.p64 q8, d7, d7 + veor.8 q11, q11, q12 + vext.8 q12, q2, q11, #8 + vext.8 q11, q11, q2, #8 + veor.8 q7, q7, q12 + veor.8 q8, q8, q11 + vshr.u32 q6, q7, #31 + vshr.u32 q9, q8, #31 + vshl.i32 q7, q7, #1 + vshl.i32 q8, q8, #1 + vext.8 q10, q6, q2, #12 + vext.8 q9, q2, q9, #12 + vext.8 q6, q2, q6, #12 + veor.8 q7, q7, q6 + veor.8 q8, q8, q9 + veor.8 q8, q8, q10 + vshl.i32 q6, q7, #31 + vshl.i32 q9, q7, #30 + vshl.i32 q10, q7, #25 + veor.8 q6, q6, q9 + veor.8 q6, q6, q10 + vext.8 q9, q6, q2, #4 + vext.8 q6, q2, q6, #4 + veor.8 q7, q7, q6 + vshr.u32 q11, q7, #1 + vshr.u32 q12, q7, #2 + vshr.u32 q13, q7, #7 + veor.8 q11, q11, q12 + veor.8 q11, q11, q13 + veor.8 q11, q11, q9 + veor.8 q7, q7, q11 + veor.8 q5, q8, q7 + lsr r12, r3, #2 + cmp r12, #0 + beq L_AES_GCMSIV_polyval_crypto_rem_start +L_AES_GCMSIV_polyval_crypto_group: + vld1.8 {q6}, [r2]! + veor.8 q6, q6, q0 + vmull.p64 q9, d12, d10 + vmull.p64 q11, d12, d11 + vmull.p64 q12, d13, d10 + vmull.p64 q10, d13, d11 + veor.8 q11, q11, q12 + vext.8 q12, q2, q11, #8 + vext.8 q11, q11, q2, #8 + veor.8 q9, q9, q12 + veor.8 q10, q10, q11 + vld1.8 {q6}, [r2]! + vmull.p64 q7, d12, d8 + vmull.p64 q11, d12, d9 + vmull.p64 q12, d13, d8 + vmull.p64 q8, d13, d9 + veor.8 q11, q11, q12 + vext.8 q12, q2, q11, #8 + vext.8 q11, q11, q2, #8 + veor.8 q7, q7, q12 + veor.8 q8, q8, q11 + veor.8 q9, q9, q7 + veor.8 q10, q10, q8 + vld1.8 {q6}, [r2]! + vmull.p64 q7, d12, d6 + vmull.p64 q11, d12, d7 + vmull.p64 q12, d13, d6 + vmull.p64 q8, d13, d7 + veor.8 q11, q11, q12 + vext.8 q12, q2, q11, #8 + vext.8 q11, q11, q2, #8 + veor.8 q7, q7, q12 + veor.8 q8, q8, q11 + veor.8 q9, q9, q7 + veor.8 q10, q10, q8 + vld1.8 {q6}, [r2]! + vmull.p64 q7, d12, d2 + vmull.p64 q11, d12, d3 + vmull.p64 q12, d13, d2 + vmull.p64 q8, d13, d3 + veor.8 q11, q11, q12 + vext.8 q12, q2, q11, #8 + vext.8 q11, q11, q2, #8 + veor.8 q7, q7, q12 + veor.8 q8, q8, q11 + veor.8 q9, q9, q7 + veor.8 q10, q10, q8 + vshr.u32 q6, q9, #31 + vshr.u32 q7, q10, #31 + vshl.i32 q9, q9, #1 + vshl.i32 q10, q10, #1 + vext.8 q8, q6, q2, #12 + vext.8 q7, q2, q7, #12 + vext.8 q6, q2, q6, #12 + veor.8 q9, q9, q6 + veor.8 q10, q10, q7 + veor.8 q10, q10, q8 + vshl.i32 q6, q9, #31 + vshl.i32 q7, q9, #30 + vshl.i32 q8, q9, #25 + veor.8 q6, q6, q7 + veor.8 q6, q6, q8 + vext.8 q7, q6, q2, #4 + vext.8 q6, q2, q6, #4 + veor.8 q9, q9, q6 + vshr.u32 q11, q9, #1 + vshr.u32 q12, q9, #2 + vshr.u32 q13, q9, #7 + veor.8 q11, q11, q12 + veor.8 q11, q11, q13 + veor.8 q11, q11, q7 + veor.8 q9, q9, q11 + veor.8 q0, q10, q9 + subs r12, r12, #1 + bne L_AES_GCMSIV_polyval_crypto_group +L_AES_GCMSIV_polyval_crypto_rem_start: + and r3, r3, #3 + cmp r3, #0 + beq L_AES_GCMSIV_polyval_crypto_done +L_AES_GCMSIV_polyval_crypto_rem: + vld1.8 {q6}, [r2]! + veor.8 q0, q0, q6 + vmull.p64 q9, d0, d2 + vmull.p64 q11, d0, d3 + vmull.p64 q12, d1, d2 + vmull.p64 q10, d1, d3 + veor.8 q11, q11, q12 + vext.8 q12, q2, q11, #8 + vext.8 q11, q11, q2, #8 + veor.8 q9, q9, q12 + veor.8 q10, q10, q11 + vshr.u32 q6, q9, #31 + vshr.u32 q7, q10, #31 + vshl.i32 q9, q9, #1 + vshl.i32 q10, q10, #1 + vext.8 q8, q6, q2, #12 + vext.8 q7, q2, q7, #12 + vext.8 q6, q2, q6, #12 + veor.8 q9, q9, q6 + veor.8 q10, q10, q7 + veor.8 q10, q10, q8 + vshl.i32 q6, q9, #31 + vshl.i32 q7, q9, #30 + vshl.i32 q8, q9, #25 + veor.8 q6, q6, q7 + veor.8 q6, q6, q8 + vext.8 q7, q6, q2, #4 + vext.8 q6, q2, q6, #4 + veor.8 q9, q9, q6 + vshr.u32 q11, q9, #1 + vshr.u32 q12, q9, #2 + vshr.u32 q13, q9, #7 + veor.8 q11, q11, q12 + veor.8 q11, q11, q13 + veor.8 q11, q11, q7 + veor.8 q9, q9, q11 + veor.8 q0, q10, q9 + subs r3, r3, #1 + bne L_AES_GCMSIV_polyval_crypto_rem +L_AES_GCMSIV_polyval_crypto_done: + vrev64.8 q0, q0 + vext.8 q0, q0, q0, #8 + vst1.8 {q0}, [r0] + vpop {d8-d15} + bx lr + .size AES_GCMSIV_polyval_crypto,.-AES_GCMSIV_polyval_crypto + .text + .align 4 + .globl AES_GCMSIV_ctr_crypto + .type AES_GCMSIV_ctr_crypto, %function +AES_GCMSIV_ctr_crypto: + push {r4, r5, r6, r7, r8, r9, r10, lr} + vpush {d8-d11} + ldr r12, [sp, #64] + ldr lr, [sp, #68] + vld1.8 {q0}, [lr] + lsr r5, r2, #4 + vmov r7, r8, d0 + vmov r9, r10, d1 +L_AES_GCMSIV_ctr_crypto_loop2: + cmp r5, #2 + blt L_AES_GCMSIV_ctr_crypto_tail + vmov d2, r7, r8 + vmov d3, r9, r10 + add r2, r7, #1 + vmov d4, r2, r8 + vmov d5, r9, r10 + add r7, r7, #2 + mov r4, r3 + sub r6, r12, #1 +L_AES_GCMSIV_ctr_crypto_rounds2: + vld1.32 {q5}, [r4]! + aese.8 q1, q5 + aesmc.8 q1, q1 + aese.8 q2, q5 + aesmc.8 q2, q2 + subs r6, r6, #1 + bne L_AES_GCMSIV_ctr_crypto_rounds2 + vld1.32 {q5}, [r4]! + aese.8 q1, q5 + aese.8 q2, q5 + vld1.32 {q5}, [r4] + veor.8 q1, q1, q5 + veor.8 q2, q2, q5 + vld1.8 {q3-q4}, [r0]! + veor.8 q3, q3, q1 + veor.8 q4, q4, q2 + vst1.8 {q3-q4}, [r1]! + sub r5, r5, #2 + b L_AES_GCMSIV_ctr_crypto_loop2 +L_AES_GCMSIV_ctr_crypto_tail: + cmp r5, #0 + beq L_AES_GCMSIV_ctr_crypto_done + vmov d2, r7, r8 + vmov d3, r9, r10 + add r7, r7, #1 + mov r4, r3 + sub r6, r12, #1 +L_AES_GCMSIV_ctr_crypto_rounds1: + vld1.32 {q5}, [r4]! + aese.8 q1, q5 + aesmc.8 q1, q1 + subs r6, r6, #1 + bne L_AES_GCMSIV_ctr_crypto_rounds1 + vld1.32 {q5}, [r4]! + aese.8 q1, q5 + vld1.32 {q5}, [r4] + veor.8 q1, q1, q5 + vld1.8 {q3}, [r0]! + veor.8 q3, q3, q1 + vst1.8 {q3}, [r1]! +L_AES_GCMSIV_ctr_crypto_done: + vmov d0, r7, r8 + vst1.8 {q0}, [lr] + vpop {d8-d11} + pop {r4, r5, r6, r7, r8, r9, r10, pc} + .size AES_GCMSIV_ctr_crypto,.-AES_GCMSIV_ctr_crypto #endif /* WOLFSSL_AESGCM_SIV */ #endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */ #endif /* !NO_AES */ diff --git a/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c b/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c index a046e48562..5dbcd6ca02 100644 --- a/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c @@ -50,9886 +50,1099 @@ #ifndef NO_AES #include -#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO +#if !defined(WOLFSSL_ARMASM_NO_BASE_IMPL) || \ + defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +#ifdef HAVE_AES_DECRYPT +XALIGNED(8) static const word32 L_AES_ARM32_td_data[] = { + 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e, + 0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303, + 0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c, + 0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3, + 0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0, + 0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9, + 0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259, + 0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8, + 0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971, + 0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a, + 0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f, + 0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b, + 0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8, + 0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab, + 0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708, + 0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682, + 0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2, + 0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe, + 0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb, + 0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10, + 0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd, + 0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015, + 0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e, + 0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee, + 0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000, + 0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72, + 0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39, + 0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e, + 0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91, + 0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a, + 0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17, + 0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9, + 0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60, + 0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e, + 0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1, + 0xcad731dc, 0x10426385, 0x40139722, 0x2084c611, + 0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1, + 0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3, + 0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964, + 0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390, + 0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b, + 0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf, + 0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46, + 0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af, + 0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512, + 0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb, + 0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a, + 0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8, + 0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c, + 0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266, + 0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8, + 0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6, + 0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604, + 0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551, + 0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41, + 0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647, + 0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c, + 0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1, + 0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737, + 0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db, + 0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340, + 0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95, + 0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1, + 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857, +}; + +#endif /* HAVE_AES_DECRYPT */ +#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AES_CBC) || \ + defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ + defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) +XALIGNED(8) static const word32 L_AES_ARM32_te_data[] = { + 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, + 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, + 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, + 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, + 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, + 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, + 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, + 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, + 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, + 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, + 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, + 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, + 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, + 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, + 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, + 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, + 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, + 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, + 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, + 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, + 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, + 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, + 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, + 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, + 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, + 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, + 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, + 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, + 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, + 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, + 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, + 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, + 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, + 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, + 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, + 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, + 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, + 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, + 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, + 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, + 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, + 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, + 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, + 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, + 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, + 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, + 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, + 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, + 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, + 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, + 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, + 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, + 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, + 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, + 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, + 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, + 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, + 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, + 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, + 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, + 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, + 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, + 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, + 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, +}; + +#endif /* HAVE_AES_DECRYPT || HAVE_AES_CBC || HAVE_AESCCM || HAVE_AESGCM || + * WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */ +#ifdef HAVE_AES_DECRYPT +static const word32* L_AES_ARM32_td = L_AES_ARM32_td_data; +#endif /* HAVE_AES_DECRYPT */ +#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AES_CBC) || \ + defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ + defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) +static const word32* L_AES_ARM32_te = L_AES_ARM32_te_data; +#endif /* HAVE_AES_DECRYPT || HAVE_AES_CBC || HAVE_AESCCM || HAVE_AESGCM || + * WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */ +#ifdef HAVE_AES_DECRYPT +void AES_invert_key(unsigned char* ks_p, word32 rounds_p); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_set_key_AARCH32(const byte* userKey_p, - int keylen_p, byte* key_p, int dir_p) +WC_OMIT_FRAME_POINTER void AES_invert_key(unsigned char* ks_p, word32 rounds_p) #else -WC_OMIT_FRAME_POINTER void AES_set_key_AARCH32(const byte* userKey, int keylen, - byte* key, int dir) +WC_OMIT_FRAME_POINTER void AES_invert_key(unsigned char* ks, word32 rounds) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* userKey __asm__ ("r0") = (const byte*)userKey_p; - register int keylen __asm__ ("r1") = (int)keylen_p; - register byte* key __asm__ ("r2") = (byte*)key_p; - register int dir __asm__ ("r3") = (int)dir_p; + register unsigned char* ks __asm__ ("r0") = (unsigned char*)ks_p; + register word32 rounds __asm__ ("r1") = (word32)rounds_p; + register word32* L_AES_ARM32_te_c __asm__ ("r2") = (word32*)L_AES_ARM32_te; + register word32* L_AES_ARM32_td_c __asm__ ("r3") = (word32*)L_AES_ARM32_td; +#else + register word32* L_AES_ARM32_te_c = (word32*)L_AES_ARM32_te; + register word32* L_AES_ARM32_td_c = (word32*)L_AES_ARM32_td; #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ __asm__ __volatile__ ( - "cmp %[keylen], #24\n\t" - "blt L_aes_set_key_arm32_crypto_start_128_%=\n\t" - "bgt L_aes_set_key_arm32_crypto_start_256_%=\n\t" - "ldr r4, [%[userKey]], #4\n\t" - "ldr r5, [%[userKey]], #4\n\t" - "ldr r6, [%[userKey]], #4\n\t" - "ldr r7, [%[userKey]], #4\n\t" - "ldr r8, [%[userKey]], #4\n\t" - "ldr r9, [%[userKey]], #4\n\t" + "mov r12, %[L_AES_ARM32_te]\n\t" + "mov lr, %[L_AES_ARM32_td]\n\t" + "add r10, %[ks], %[rounds], lsl #4\n\t" + "mov r11, %[rounds]\n\t" + "\n" + "L_AES_invert_key_loop_%=:\n\t" + "ldm %[ks], {r2, r3, r4, r5}\n\t" + "ldm r10, {r6, r7, r8, r9}\n\t" + "stm r10, {r2, r3, r4, r5}\n\t" + "stm %[ks]!, {r6, r7, r8, r9}\n\t" + "subs r11, r11, #2\n\t" + "sub r10, r10, #16\n\t" + "bne L_AES_invert_key_loop_%=\n\t" + "sub %[ks], %[ks], %[rounds], lsl #3\n\t" + "add %[ks], %[ks], #16\n\t" + "sub r11, %[rounds], #1\n\t" + "\n" + "L_AES_invert_key_mix_loop_%=:\n\t" + "ldm %[ks], {r2, r3, r4, r5}\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r2, #24\n\t" + "lsr r6, r6, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r6, r2\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r6, r2, #0, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r2, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "uxtb r7, r2, ror #8\n\t" #endif - "vdup.32 q1, r9\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #1\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "eor r8, r8, r7\n\t" - "eor r9, r9, r8\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "ubfx r7, r2, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r8, r2, #8\n\t" + "lsr r8, r8, #24\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "uxtb r8, r2, ror #16\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" -#endif - "vdup.32 q1, r9\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #2\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "eor r8, r8, r7\n\t" - "eor r9, r9, r8\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" -#else - "strd r4, r5, [%[key]], #8\n\t" + "ubfx r8, r2, #16, #8\n\t" #endif + "lsr r9, r2, #24\n\t" + "ldrb r6, [r12, r6, lsl #2]\n\t" + "ldrb r7, [r12, r7, lsl #2]\n\t" + "ldrb r8, [r12, r8, lsl #2]\n\t" + "ldrb r9, [r12, r9, lsl #2]\n\t" + "ldr r6, [lr, r6, lsl #2]\n\t" + "ldr r7, [lr, r7, lsl #2]\n\t" + "ldr r8, [lr, r8, lsl #2]\n\t" + "ldr r9, [lr, r9, lsl #2]\n\t" + "eor r8, r8, r6, ror #16\n\t" + "eor r8, r8, r7, ror #8\n\t" + "eor r8, r8, r9, ror #24\n\t" + "str r8, [%[ks]], #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r3, #24\n\t" + "lsr r6, r6, #24\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "uxtb r6, r3\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "ubfx r6, r3, #0, #8\n\t" #endif - "vdup.32 q1, r9\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #4\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "eor r8, r8, r7\n\t" - "eor r9, r9, r8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r3, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r7, r3, ror #8\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r7, r3, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r8, r3, #8\n\t" + "lsr r8, r8, #24\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "uxtb r8, r3, ror #16\n\t" #endif - "vdup.32 q1, r9\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #8\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "eor r8, r8, r7\n\t" - "eor r9, r9, r8\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "ubfx r8, r3, #16, #8\n\t" #endif + "lsr r9, r3, #24\n\t" + "ldrb r6, [r12, r6, lsl #2]\n\t" + "ldrb r7, [r12, r7, lsl #2]\n\t" + "ldrb r8, [r12, r8, lsl #2]\n\t" + "ldrb r9, [r12, r9, lsl #2]\n\t" + "ldr r6, [lr, r6, lsl #2]\n\t" + "ldr r7, [lr, r7, lsl #2]\n\t" + "ldr r8, [lr, r8, lsl #2]\n\t" + "ldr r9, [lr, r9, lsl #2]\n\t" + "eor r8, r8, r6, ror #16\n\t" + "eor r8, r8, r7, ror #8\n\t" + "eor r8, r8, r9, ror #24\n\t" + "str r8, [%[ks]], #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r4, #24\n\t" + "lsr r6, r6, #24\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "uxtb r6, r4\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "ubfx r6, r4, #0, #8\n\t" #endif - "vdup.32 q1, r9\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #16\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "eor r8, r8, r7\n\t" - "eor r9, r9, r8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r4, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r7, r4, ror #8\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r7, r4, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r8, r4, #8\n\t" + "lsr r8, r8, #24\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "uxtb r8, r4, ror #16\n\t" #endif - "vdup.32 q1, r9\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #32\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "eor r8, r8, r7\n\t" - "eor r9, r9, r8\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "ubfx r8, r4, #16, #8\n\t" #endif + "lsr r9, r4, #24\n\t" + "ldrb r6, [r12, r6, lsl #2]\n\t" + "ldrb r7, [r12, r7, lsl #2]\n\t" + "ldrb r8, [r12, r8, lsl #2]\n\t" + "ldrb r9, [r12, r9, lsl #2]\n\t" + "ldr r6, [lr, r6, lsl #2]\n\t" + "ldr r7, [lr, r7, lsl #2]\n\t" + "ldr r8, [lr, r8, lsl #2]\n\t" + "ldr r9, [lr, r9, lsl #2]\n\t" + "eor r8, r8, r6, ror #16\n\t" + "eor r8, r8, r7, ror #8\n\t" + "eor r8, r8, r9, ror #24\n\t" + "str r8, [%[ks]], #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r5, #24\n\t" + "lsr r6, r6, #24\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "uxtb r6, r5\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "ubfx r6, r5, #0, #8\n\t" #endif - "vdup.32 q1, r9\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #0x40\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "eor r8, r8, r7\n\t" - "eor r9, r9, r8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r5, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r7, r5, ror #8\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r7, r5, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r8, r5, #8\n\t" + "lsr r8, r8, #24\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "uxtb r8, r5, ror #16\n\t" #endif - "vdup.32 q1, r9\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #0x80\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "ubfx r8, r5, #16, #8\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" + "lsr r9, r5, #24\n\t" + "ldrb r6, [r12, r6, lsl #2]\n\t" + "ldrb r7, [r12, r7, lsl #2]\n\t" + "ldrb r8, [r12, r8, lsl #2]\n\t" + "ldrb r9, [r12, r9, lsl #2]\n\t" + "ldr r6, [lr, r6, lsl #2]\n\t" + "ldr r7, [lr, r7, lsl #2]\n\t" + "ldr r8, [lr, r8, lsl #2]\n\t" + "ldr r9, [lr, r9, lsl #2]\n\t" + "eor r8, r8, r6, ror #16\n\t" + "eor r8, r8, r7, ror #8\n\t" + "eor r8, r8, r9, ror #24\n\t" + "str r8, [%[ks]], #4\n\t" + "subs r11, r11, #1\n\t" + "bne L_AES_invert_key_mix_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [ks] "+r" (ks), [rounds] "+r" (rounds), + [L_AES_ARM32_te] "+r" (L_AES_ARM32_te_c), + [L_AES_ARM32_td] "+r" (L_AES_ARM32_td_c) + : #else - "strd r6, r7, [%[key]], #8\n\t" -#endif - "cmp %[dir], #0\n\t" - "beq L_aes_set_key_arm32_crypto_done_%=\n\t" - "sub %[key], %[key], #0xd0\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0xc0\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0xc0\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0xc0\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0xb0\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0xa0\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0xa0\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0xa0\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0x90\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0x80\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0x80\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0x80\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0x70\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0x60\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0x60\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0x60\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0x50\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0x40\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0x40\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0x40\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #48\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #32\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #32\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #32\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #16\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "aesimc.8 q0, q0\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "b L_aes_set_key_arm32_crypto_done_%=\n\t" - "\n" - "L_aes_set_key_arm32_crypto_start_256_%=:\n\t" - "ldr r4, [%[userKey]], #4\n\t" - "ldr r5, [%[userKey]], #4\n\t" - "ldr r6, [%[userKey]], #4\n\t" - "ldr r7, [%[userKey]], #4\n\t" - "ldr r8, [%[userKey]], #4\n\t" - "ldr r9, [%[userKey]], #4\n\t" - "ldr r10, [%[userKey]], #4\n\t" - "ldr r11, [%[userKey]], #4\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" + : + : [ks] "r" (ks), [rounds] "r" (rounds), + [L_AES_ARM32_te] "r" (L_AES_ARM32_te_c), + [L_AES_ARM32_td] "r" (L_AES_ARM32_td_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "lr", "r4", "r5", "r6", "r7", "r8", "r9", + "r10", "r11" + ); +} + +#endif /* HAVE_AES_DECRYPT */ +XALIGNED(8) static const word32 L_AES_ARM32_rcon[] = { + 0x01000000, 0x02000000, 0x04000000, 0x08000000, + 0x10000000, 0x20000000, 0x40000000, 0x80000000, + 0x1b000000, 0x36000000 +}; + +void AES_set_encrypt_key(const unsigned char* key_p, word32 len_p, + unsigned char* ks_p); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_set_encrypt_key(const unsigned char* key_p, + word32 len_p, unsigned char* ks_p) #else - "strd r4, r5, [%[key]], #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" +WC_OMIT_FRAME_POINTER void AES_set_encrypt_key(const unsigned char* key, + word32 len, unsigned char* ks) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const unsigned char* key __asm__ ("r0") = + (const unsigned char*)key_p; + register word32 len __asm__ ("r1") = (word32)len_p; + register unsigned char* ks __asm__ ("r2") = (unsigned char*)ks_p; + register word32* L_AES_ARM32_te_c __asm__ ("r3") = (word32*)L_AES_ARM32_te; + register word32* L_AES_ARM32_rcon_c __asm__ ("r12") = + (word32*)&L_AES_ARM32_rcon; #else - "strd r6, r7, [%[key]], #8\n\t" -#endif + register word32* L_AES_ARM32_te_c = (word32*)L_AES_ARM32_te; + register word32* L_AES_ARM32_rcon_c = (word32*)&L_AES_ARM32_rcon; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_ARM32_rcon]}\n\t" + "mov r8, %[L_AES_ARM32_te]\n\t" + "mov lr, %[L_AES_ARM32_rcon]\n\t" + "cmp %[len], #0x80\n\t" + "beq L_AES_set_encrypt_key_start_128_%=\n\t" + "cmp %[len], #0xc0\n\t" + "beq L_AES_set_encrypt_key_start_192_%=\n\t" + "ldr r4, [%[key]]\n\t" + "ldr r5, [%[key], #4]\n\t" + "ldr r6, [%[key], #8]\n\t" + "ldr r7, [%[key], #12]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor r3, r4, r4, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, r3, lsr #8\n\t" + /* REV r5, r5 */ + "eor r3, r5, r5, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, r3, lsr #8\n\t" + /* REV r6, r6 */ + "eor r3, r6, r6, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, r3, lsr #8\n\t" + /* REV r7, r7 */ + "eor r3, r7, r7, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, r3, lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "stm %[ks]!, {r4, r5, r6, r7}\n\t" + "ldr r4, [%[key], #16]\n\t" + "ldr r5, [%[key], #20]\n\t" + "ldr r6, [%[key], #24]\n\t" + "ldr r7, [%[key], #28]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor r3, r4, r4, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, r3, lsr #8\n\t" + /* REV r5, r5 */ + "eor r3, r5, r5, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, r3, lsr #8\n\t" + /* REV r6, r6 */ + "eor r3, r6, r6, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, r3, lsr #8\n\t" + /* REV r7, r7 */ + "eor r3, r7, r7, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, r3, lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "stm %[ks], {r4, r5, r6, r7}\n\t" + "sub %[ks], %[ks], #16\n\t" + "mov r12, #6\n\t" + "\n" + "L_AES_set_encrypt_key_loop_256_%=:\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r7, #24\n\t" + "lsr r4, r4, #24\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "uxtb r4, r7\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r10, r11}\n\t" #else - "strd r10, r11, [%[key]], #8\n\t" + "ubfx r4, r7, #0, #8\n\t" #endif - "vdup.32 q1, r11\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #1\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "eor r8, r8, r12\n\t" - "eor r9, r9, r8\n\t" - "eor r10, r10, r9\n\t" - "eor r11, r11, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r5, r7, #16\n\t" + "lsr r5, r5, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r5, r7, ror #8\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r5, r7, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r7, #8\n\t" + "lsr r6, r6, #24\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "uxtb r6, r7, ror #16\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r10, r11}\n\t" #else - "strd r10, r11, [%[key]], #8\n\t" + "ubfx r6, r7, #16, #8\n\t" #endif - "vdup.32 q1, r11\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #2\n\t" - "eor r4, r4, r12\n\t" + "lsr r7, r7, #24\n\t" + "ldrb r4, [r8, r4, lsl #2]\n\t" + "ldrb r5, [r8, r5, lsl #2]\n\t" + "ldrb r6, [r8, r6, lsl #2]\n\t" + "ldrb r7, [r8, r7, lsl #2]\n\t" + "eor r3, r7, r4, lsl #8\n\t" + "eor r3, r3, r5, lsl #16\n\t" + "eor r3, r3, r6, lsl #24\n\t" + "ldm %[ks]!, {r4, r5, r6, r7}\n\t" + "eor r4, r4, r3\n\t" + "ldm lr!, {r3}\n\t" + "eor r4, r4, r3\n\t" "eor r5, r5, r4\n\t" "eor r6, r6, r5\n\t" "eor r7, r7, r6\n\t" - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "eor r8, r8, r12\n\t" - "eor r9, r9, r8\n\t" - "eor r10, r10, r9\n\t" - "eor r11, r11, r10\n\t" + "add %[ks], %[ks], #16\n\t" + "stm %[ks], {r4, r5, r6, r7}\n\t" + "sub %[ks], %[ks], #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r7, #16\n\t" + "lsr r4, r4, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r4, r7, ror #8\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r4, r7, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" -#else - "strd r8, r9, [%[key]], #8\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r5, r7, #8\n\t" + "lsr r5, r5, #24\n\t" +#else + "uxtb r5, r7, ror #16\n\t" +#endif +#else + "ubfx r5, r7, #16, #8\n\t" #endif + "lsr r6, r7, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r10, r11}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r3, r7, #24\n\t" + "lsr r3, r3, #24\n\t" #else - "strd r10, r11, [%[key]], #8\n\t" + "uxtb r3, r7\n\t" #endif - "vdup.32 q1, r11\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #4\n\t" - "eor r4, r4, r12\n\t" +#else + "ubfx r3, r7, #0, #8\n\t" +#endif + "ldrb r4, [r8, r4, lsl #2]\n\t" + "ldrb r6, [r8, r6, lsl #2]\n\t" + "ldrb r5, [r8, r5, lsl #2]\n\t" + "ldrb r3, [r8, r3, lsl #2]\n\t" + "eor r3, r3, r4, lsl #8\n\t" + "eor r3, r3, r5, lsl #16\n\t" + "eor r3, r3, r6, lsl #24\n\t" + "ldm %[ks]!, {r4, r5, r6, r7}\n\t" + "eor r4, r4, r3\n\t" "eor r5, r5, r4\n\t" "eor r6, r6, r5\n\t" "eor r7, r7, r6\n\t" - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "eor r8, r8, r12\n\t" - "eor r9, r9, r8\n\t" - "eor r10, r10, r9\n\t" - "eor r11, r11, r10\n\t" + "add %[ks], %[ks], #16\n\t" + "stm %[ks], {r4, r5, r6, r7}\n\t" + "sub %[ks], %[ks], #16\n\t" + "subs r12, r12, #1\n\t" + "bne L_AES_set_encrypt_key_loop_256_%=\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r7, #24\n\t" + "lsr r4, r4, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r4, r7\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r4, r7, #0, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r5, r7, #16\n\t" + "lsr r5, r5, #24\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "uxtb r5, r7, ror #8\n\t" +#endif +#else + "ubfx r5, r7, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r10, r11}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r7, #8\n\t" + "lsr r6, r6, #24\n\t" #else - "strd r10, r11, [%[key]], #8\n\t" + "uxtb r6, r7, ror #16\n\t" #endif - "vdup.32 q1, r11\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #8\n\t" - "eor r4, r4, r12\n\t" +#else + "ubfx r6, r7, #16, #8\n\t" +#endif + "lsr r7, r7, #24\n\t" + "ldrb r4, [r8, r4, lsl #2]\n\t" + "ldrb r5, [r8, r5, lsl #2]\n\t" + "ldrb r6, [r8, r6, lsl #2]\n\t" + "ldrb r7, [r8, r7, lsl #2]\n\t" + "eor r3, r7, r4, lsl #8\n\t" + "eor r3, r3, r5, lsl #16\n\t" + "eor r3, r3, r6, lsl #24\n\t" + "ldm %[ks]!, {r4, r5, r6, r7}\n\t" + "eor r4, r4, r3\n\t" + "ldm lr!, {r3}\n\t" + "eor r4, r4, r3\n\t" "eor r5, r5, r4\n\t" "eor r6, r6, r5\n\t" "eor r7, r7, r6\n\t" - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "eor r8, r8, r12\n\t" - "eor r9, r9, r8\n\t" - "eor r10, r10, r9\n\t" - "eor r11, r11, r10\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" + "add %[ks], %[ks], #16\n\t" + "stm %[ks], {r4, r5, r6, r7}\n\t" + "sub %[ks], %[ks], #16\n\t" + "b L_AES_set_encrypt_key_end_%=\n\t" + "\n" + "L_AES_set_encrypt_key_start_192_%=:\n\t" + "ldr r4, [%[key]]\n\t" + "ldr r5, [%[key], #4]\n\t" + "ldr r6, [%[key], #8]\n\t" + "ldr r7, [%[key], #12]\n\t" + "ldr %[len], [%[key], #20]\n\t" + "ldr %[key], [%[key], #16]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor r3, r4, r4, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, r3, lsr #8\n\t" + /* REV r5, r5 */ + "eor r3, r5, r5, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, r3, lsr #8\n\t" + /* REV r6, r6 */ + "eor r3, r6, r6, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, r3, lsr #8\n\t" + /* REV r7, r7 */ + "eor r3, r7, r7, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, r3, lsr #8\n\t" + /* REV r0, r0 */ + "eor r3, %[key], %[key], ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror %[key], %[key], #8\n\t" + "eor %[key], %[key], r3, lsr #8\n\t" + /* REV r1, r1 */ + "eor r3, %[len], %[len], ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror %[len], %[len], #8\n\t" + "eor %[len], %[len], r3, lsr #8\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" -#endif + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" + "rev %[key], %[key]\n\t" + "rev %[len], %[len]\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "stm %[ks], {r4, r5, r6, r7}\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" + "str %[key], [%[ks], #16]\n\t" + "str %[len], [%[ks], #20]\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "strd %[key], %[len], [%[ks], #16]\n\t" #endif + "mov r7, %[len]\n\t" + "mov r12, #7\n\t" + "\n" + "L_AES_set_encrypt_key_loop_192_%=:\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r0, r7, #24\n\t" + "lsr r0, r0, #24\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "uxtb r0, r7\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r10, r11}\n\t" #else - "strd r10, r11, [%[key]], #8\n\t" + "ubfx r0, r7, #0, #8\n\t" #endif - "vdup.32 q1, r11\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #16\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "eor r8, r8, r12\n\t" - "eor r9, r9, r8\n\t" - "eor r10, r10, r9\n\t" - "eor r11, r11, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r1, r7, #16\n\t" + "lsr r1, r1, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r1, r7, ror #8\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r1, r7, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r7, #8\n\t" + "lsr r4, r4, #24\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "uxtb r4, r7, ror #16\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r10, r11}\n\t" #else - "strd r10, r11, [%[key]], #8\n\t" + "ubfx r4, r7, #16, #8\n\t" #endif - "vdup.32 q1, r11\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #32\n\t" - "eor r4, r4, r12\n\t" + "lsr r7, r7, #24\n\t" + "ldrb r0, [r8, r0, lsl #2]\n\t" + "ldrb r1, [r8, r1, lsl #2]\n\t" + "ldrb r4, [r8, r4, lsl #2]\n\t" + "ldrb r7, [r8, r7, lsl #2]\n\t" + "eor r3, r7, r0, lsl #8\n\t" + "eor r3, r3, r1, lsl #16\n\t" + "eor r3, r3, r4, lsl #24\n\t" + "ldm %[ks]!, {r0, r1, r4, r5, r6, r7}\n\t" + "eor r0, r0, r3\n\t" + "ldm lr!, {r3}\n\t" + "eor r0, r0, r3\n\t" + "eor r1, r1, r0\n\t" + "eor r4, r4, r1\n\t" "eor r5, r5, r4\n\t" "eor r6, r6, r5\n\t" "eor r7, r7, r6\n\t" - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "eor r8, r8, r12\n\t" - "eor r9, r9, r8\n\t" - "eor r10, r10, r9\n\t" - "eor r11, r11, r10\n\t" + "stm %[ks], {r0, r1, r4, r5, r6, r7}\n\t" + "subs r12, r12, #1\n\t" + "bne L_AES_set_encrypt_key_loop_192_%=\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r0, r7, #24\n\t" + "lsr r0, r0, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r0, r7\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r0, r7, #0, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r8, r9}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r1, r7, #16\n\t" + "lsr r1, r1, #24\n\t" #else - "strd r8, r9, [%[key]], #8\n\t" + "uxtb r1, r7, ror #8\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r10, r11}\n\t" #else - "strd r10, r11, [%[key]], #8\n\t" + "ubfx r1, r7, #8, #8\n\t" #endif - "vdup.32 q1, r11\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #0x40\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r7, #8\n\t" + "lsr r4, r4, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r4, r7, ror #16\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r4, r7, #16, #8\n\t" #endif - "cmp %[dir], #0\n\t" - "beq L_aes_set_key_arm32_crypto_done_%=\n\t" - "sub %[key], %[key], #0xf0\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0xe0\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0xe0\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0xe0\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0xd0\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0xc0\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0xc0\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0xc0\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0xb0\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0xa0\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0xa0\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0xa0\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0x90\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0x80\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0x80\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0x80\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0x70\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0x60\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0x60\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0x60\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0x50\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0x40\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0x40\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0x40\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #48\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #32\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #32\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #32\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #16\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "aesimc.8 q0, q0\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "b L_aes_set_key_arm32_crypto_done_%=\n\t" + "lsr r7, r7, #24\n\t" + "ldrb r0, [r8, r0, lsl #2]\n\t" + "ldrb r1, [r8, r1, lsl #2]\n\t" + "ldrb r4, [r8, r4, lsl #2]\n\t" + "ldrb r7, [r8, r7, lsl #2]\n\t" + "eor r3, r7, r0, lsl #8\n\t" + "eor r3, r3, r1, lsl #16\n\t" + "eor r3, r3, r4, lsl #24\n\t" + "ldm %[ks]!, {r0, r1, r4, r5, r6, r7}\n\t" + "eor r0, r0, r3\n\t" + "ldm lr!, {r3}\n\t" + "eor r0, r0, r3\n\t" + "eor r1, r1, r0\n\t" + "eor r4, r4, r1\n\t" + "eor r5, r5, r4\n\t" + "stm %[ks], {r0, r1, r4, r5}\n\t" + "b L_AES_set_encrypt_key_end_%=\n\t" "\n" - "L_aes_set_key_arm32_crypto_start_128_%=:\n\t" - "ldr r4, [%[userKey]], #4\n\t" - "ldr r5, [%[userKey]], #4\n\t" - "ldr r6, [%[userKey]], #4\n\t" - "ldr r7, [%[userKey]], #4\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" -#else - "strd r4, r5, [%[key]], #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" + "L_AES_set_encrypt_key_start_128_%=:\n\t" + "ldr r4, [%[key]]\n\t" + "ldr r5, [%[key], #4]\n\t" + "ldr r6, [%[key], #8]\n\t" + "ldr r7, [%[key], #12]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor r3, r4, r4, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, r3, lsr #8\n\t" + /* REV r5, r5 */ + "eor r3, r5, r5, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, r3, lsr #8\n\t" + /* REV r6, r6 */ + "eor r3, r6, r6, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, r3, lsr #8\n\t" + /* REV r7, r7 */ + "eor r3, r7, r7, ror #16\n\t" + "bic r3, r3, #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, r3, lsr #8\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" -#endif - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #1\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "stm %[ks], {r4, r5, r6, r7}\n\t" + "mov r12, #10\n\t" + "\n" + "L_AES_set_encrypt_key_loop_128_%=:\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r7, #24\n\t" + "lsr r4, r4, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r4, r7\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r4, r7, #0, #8\n\t" #endif - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #2\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r5, r7, #16\n\t" + "lsr r5, r5, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r5, r7, ror #8\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r5, r7, #8, #8\n\t" #endif - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #4\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r7, #8\n\t" + "lsr r6, r6, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r6, r7, ror #16\n\t" #endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "ubfx r6, r7, #16, #8\n\t" #endif - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #8\n\t" - "eor r4, r4, r12\n\t" + "lsr r7, r7, #24\n\t" + "ldrb r4, [r8, r4, lsl #2]\n\t" + "ldrb r5, [r8, r5, lsl #2]\n\t" + "ldrb r6, [r8, r6, lsl #2]\n\t" + "ldrb r7, [r8, r7, lsl #2]\n\t" + "eor r3, r7, r4, lsl #8\n\t" + "eor r3, r3, r5, lsl #16\n\t" + "eor r3, r3, r6, lsl #24\n\t" + "ldm %[ks]!, {r4, r5, r6, r7}\n\t" + "eor r4, r4, r3\n\t" + "ldm lr!, {r3}\n\t" + "eor r4, r4, r3\n\t" "eor r5, r5, r4\n\t" "eor r6, r6, r5\n\t" "eor r7, r7, r6\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" -#else - "strd r4, r5, [%[key]], #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" + "stm %[ks], {r4, r5, r6, r7}\n\t" + "subs r12, r12, #1\n\t" + "bne L_AES_set_encrypt_key_loop_128_%=\n\t" + "\n" + "L_AES_set_encrypt_key_end_%=:\n\t" + "pop {%[L_AES_ARM32_rcon]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [key] "+r" (key), [len] "+r" (len), [ks] "+r" (ks), + [L_AES_ARM32_te] "+r" (L_AES_ARM32_te_c), + [L_AES_ARM32_rcon] "+r" (L_AES_ARM32_rcon_c) + : #else - "strd r6, r7, [%[key]], #8\n\t" -#endif - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #16\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" + : + : [key] "r" (key), [len] "r" (len), [ks] "r" (ks), + [L_AES_ARM32_te] "r" (L_AES_ARM32_te_c), + [L_AES_ARM32_rcon] "r" (L_AES_ARM32_rcon_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8" + ); +} + +#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE +void AES_encrypt_block(const word32* te_p, int nr_p, int len_p, + const word32* ks_p); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te_p, int nr_p, + int len_p, const word32* ks_p) +#else +WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te, int nr, int len, + const word32* ks) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const word32* te __asm__ ("r0") = (const word32*)te_p; + register int nr __asm__ ("r1") = (int)nr_p; + register int len __asm__ ("r2") = (int)len_p; + register const word32* ks __asm__ ("r3") = (const word32*)ks_p; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "\n" + "L_AES_encrypt_block_nr_%=:\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r8, r5, #8\n\t" + "lsr r8, r8, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r8, r5, ror #16\n\t" #endif +#else + "ubfx r8, r5, #16, #8\n\t" +#endif + "lsr r11, r4, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r6, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "uxtb lr, r6, ror #8\n\t" +#endif +#else + "ubfx lr, r6, #8, #8\n\t" #endif - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #32\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r7, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r2, r7\n\t" +#endif +#else + "ubfx r2, r7, #0, #8\n\t" #endif + "ldr r8, [%[te], r8, lsl #2]\n\t" + "ldr r11, [%[te], r11, lsl #2]\n\t" + "ldr lr, [%[te], lr, lsl #2]\n\t" + "ldr r2, [%[te], r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r9, r6, #8\n\t" + "lsr r9, r9, #24\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "uxtb r9, r6, ror #16\n\t" #endif - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #0x40\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" +#else + "ubfx r9, r6, #16, #8\n\t" +#endif + "eor r8, r8, r11, ror #24\n\t" + "lsr r11, r5, #24\n\t" + "eor r8, r8, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r7, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb lr, r7, ror #8\n\t" #endif +#else + "ubfx lr, r7, #8, #8\n\t" +#endif + "eor r8, r8, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r4, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "uxtb r2, r4\n\t" #endif - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, #0x80\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" +#else + "ubfx r2, r4, #0, #8\n\t" +#endif + "ldr r9, [%[te], r9, lsl #2]\n\t" + "ldr r11, [%[te], r11, lsl #2]\n\t" + "ldr lr, [%[te], lr, lsl #2]\n\t" + "ldr r2, [%[te], r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r10, r7, #8\n\t" + "lsr r10, r10, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r10, r7, ror #16\n\t" +#endif +#else + "ubfx r10, r7, #16, #8\n\t" #endif + "eor r9, r9, r11, ror #24\n\t" + "lsr r11, r6, #24\n\t" + "eor r9, r9, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r4, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "uxtb lr, r4, ror #8\n\t" #endif - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "mov lr, #27\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, lr\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" +#else + "ubfx lr, r4, #8, #8\n\t" +#endif + "eor r9, r9, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r5, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r2, r5\n\t" +#endif +#else + "ubfx r2, r5, #0, #8\n\t" #endif + "ldr r10, [%[te], r10, lsl #2]\n\t" + "ldr r11, [%[te], r11, lsl #2]\n\t" + "ldr lr, [%[te], lr, lsl #2]\n\t" + "ldr r2, [%[te], r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r6, #24\n\t" + "lsr r6, r6, #24\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "uxtb r6, r6\n\t" #endif - "vdup.32 q1, r7\n\t" - "vmov.i32 q0, #0\n\t" - "aese.8 q0, q1\n\t" - "vmov.i32 r12, s0\n\t" - "mov lr, #54\n\t" - "ror r12, r12, #8\n\t" - "eor r4, r4, lr\n\t" - "eor r4, r4, r12\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" +#else + "ubfx r6, r6, #0, #8\n\t" +#endif + "eor r10, r10, r11, ror #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r4, r5}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r11, r4, #8\n\t" + "lsr r11, r11, #24\n\t" #else - "strd r4, r5, [%[key]], #8\n\t" + "uxtb r11, r4, ror #16\n\t" #endif +#else + "ubfx r11, r4, #16, #8\n\t" +#endif + "eor r10, r10, lr, ror #8\n\t" + "lsr lr, r7, #24\n\t" + "eor r10, r10, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r2!, {r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r5, #16\n\t" + "lsr r2, r2, #24\n\t" #else - "strd r6, r7, [%[key]], #8\n\t" + "uxtb r2, r5, ror #8\n\t" #endif - "cmp %[dir], #0\n\t" - "beq L_aes_set_key_arm32_crypto_done_%=\n\t" - "sub %[key], %[key], #0xb0\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0xa0\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0xa0\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0xa0\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0x90\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0x80\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0x80\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0x80\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0x70\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0x60\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0x60\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0x60\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #0x50\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #0x40\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #0x40\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #0x40\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #48\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "add %[key], %[key], #32\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "sub %[key], %[key], #32\n\t" - "aesimc.8 q0, q0\n\t" - "aesimc.8 q1, q1\n\t" - "vst1.32 {q1}, [%[key]]\n\t" - "add %[key], %[key], #32\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "sub %[key], %[key], #16\n\t" - "vld1.32 {q0}, [%[key]]\n\t" - "aesimc.8 q0, q0\n\t" - "vst1.32 {q0}, [%[key]]\n\t" - "\n" - "L_aes_set_key_arm32_crypto_done_%=:\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [userKey] "+r" (userKey), [keylen] "+r" (keylen), [key] "+r" (key), - [dir] "+r" (dir) - : #else - : - : [userKey] "r" (userKey), [keylen] "r" (keylen), [key] "r" (key), - [dir] "r" (dir) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r12", "lr", "r4", "r5", "r6", "r7", "r8", "r9", - "r10", "r11", "q0", "q1" - ); -} - -#if defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ - defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || \ - defined(HAVE_AES_CBC) -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_encrypt_AARCH32(const byte* inBlock_p, - byte* outBlock_p, byte* key_p, int nr_p) + "ubfx r2, r5, #8, #8\n\t" +#endif + "ldr r6, [%[te], r6, lsl #2]\n\t" + "ldr lr, [%[te], lr, lsl #2]\n\t" + "ldr r11, [%[te], r11, lsl #2]\n\t" + "ldr r2, [%[te], r2, lsl #2]\n\t" + "eor lr, lr, r6, ror #24\n\t" + "ldm %[ks]!, {r4, r5, r6, r7}\n\t" + "eor r11, r11, lr, ror #24\n\t" + "eor r11, r11, r2, ror #8\n\t" + /* XOR in Key Schedule */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r9, #8\n\t" + "lsr r4, r4, #24\n\t" #else -WC_OMIT_FRAME_POINTER void AES_encrypt_AARCH32(const byte* inBlock, - byte* outBlock, byte* key, int nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* inBlock __asm__ ("r0") = (const byte*)inBlock_p; - register byte* outBlock __asm__ ("r1") = (byte*)outBlock_p; - register byte* key __asm__ ("r2") = (byte*)key_p; - register int nr __asm__ ("r3") = (int)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "vld1.8 {q0}, [%[inBlock]]\n\t" - "vldm %[key]!, {q1-q4}\n\t" - "aese.8 q0, q1\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q2\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "vldm %[key]!, {q1-q4}\n\t" - "aese.8 q0, q1\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q2\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "subs %[nr], %[nr], #10\n\t" - "vld1.32 {q1-q2}, [%[key]]!\n\t" - "aese.8 q0, q1\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q2\n\t" - "beq L_aes_encrypt_arm32_crypto_round_done_%=\n\t" - "vld1.32 {q1-q2}, [%[key]]!\n\t" - "subs %[nr], %[nr], #2\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q1\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q2\n\t" - "beq L_aes_encrypt_arm32_crypto_round_done_%=\n\t" - "vld1.32 {q1-q2}, [%[key]]!\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q1\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q2\n\t" - "\n" - "L_aes_encrypt_arm32_crypto_round_done_%=:\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "veor.32 q0, q0, q1\n\t" - "vst1.8 {q0}, [%[outBlock]]\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [inBlock] "+r" (inBlock), [outBlock] "+r" (outBlock), - [key] "+r" (key), [nr] "+r" (nr) - : + "uxtb r4, r9, ror #16\n\t" +#endif #else - : - : [inBlock] "r" (inBlock), [outBlock] "r" (outBlock), [key] "r" (key), - [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "q0", "q1", "q2", "q3", "q4" - ); -} - -#endif /* defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || - * defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || - * defined(HAVE_AES_CBC) */ -#if !defined(WC_AES_BITSLICED) || defined(WOLFSSL_AES_DIRECT) || \ - defined(WOLFSSL_AES_COUNTER) -#ifdef HAVE_AES_DECRYPT -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_decrypt_AARCH32(const byte* inBlock_p, - byte* outBlock_p, byte* key_p, int nr_p) + "ubfx r4, r9, #16, #8\n\t" +#endif + "lsr r7, r8, #24\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r10, #16\n\t" + "lsr lr, lr, #24\n\t" #else -WC_OMIT_FRAME_POINTER void AES_decrypt_AARCH32(const byte* inBlock, - byte* outBlock, byte* key, int nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* inBlock __asm__ ("r0") = (const byte*)inBlock_p; - register byte* outBlock __asm__ ("r1") = (byte*)outBlock_p; - register byte* key __asm__ ("r2") = (byte*)key_p; - register int nr __asm__ ("r3") = (int)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "vld1.8 {q0}, [%[inBlock]]\n\t" - "vldm %[key]!, {q1-q4}\n\t" - "aesd.8 q0, q1\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q2\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q3\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q4\n\t" - "aesimc.8 q0, q0\n\t" - "vldm %[key]!, {q1-q4}\n\t" - "aesd.8 q0, q1\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q2\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q3\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q4\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q1-q2}, [%[key]]!\n\t" - "aesd.8 q0, q1\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q2\n\t" - "subs %[nr], %[nr], #10\n\t" - "beq L_aes_decrypt_arm32_crypto_round_done_%=\n\t" - "vld1.32 {q1-q2}, [%[key]]!\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q1\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q2\n\t" - "subs %[nr], %[nr], #2\n\t" - "beq L_aes_decrypt_arm32_crypto_round_done_%=\n\t" - "vld1.32 {q1-q2}, [%[key]]!\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q1\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q2\n\t" - "\n" - "L_aes_decrypt_arm32_crypto_round_done_%=:\n\t" - "vld1.32 {q1}, [%[key]]\n\t" - "veor.32 q0, q0, q1\n\t" - "vst1.8 {q0}, [%[outBlock]]\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [inBlock] "+r" (inBlock), [outBlock] "+r" (outBlock), - [key] "+r" (key), [nr] "+r" (nr) - : + "uxtb lr, r10, ror #8\n\t" +#endif #else - : - : [inBlock] "r" (inBlock), [outBlock] "r" (outBlock), [key] "r" (key), - [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "q0", "q1", "q2", "q3", "q4" - ); -} - -#endif /* HAVE_AES_DECRYPT */ -#endif /* !defined(WC_AES_BITSLICED) || defined(WOLFSSL_AES_DIRECT) || - * defined(WOLFSSL_AES_COUNTER) */ -#ifdef HAVE_AES_ECB -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_encrypt_blocks_AARCH32(const byte* in_p, - byte* out_p, word32 sz_p, byte* key_p, int nr_p) -#else -WC_OMIT_FRAME_POINTER void AES_encrypt_blocks_AARCH32(const byte* in, byte* out, - word32 sz, byte* key, int nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* in __asm__ ("r0") = (const byte*)in_p; - register byte* out __asm__ ("r1") = (byte*)out_p; - register word32 sz __asm__ ("r2") = (word32)sz_p; - register byte* key __asm__ ("r3") = (byte*)key_p; - register int nr __asm__ ("r12") = (int)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[nr]}\n\t" - "ldr r12, [sp]\n\t" - "vldm.32 %[key]!, {q0-q7}\n\t" - "lsr %[sz], %[sz], #4\n\t" - "cmp r12, #12\n\t" - "blt L_aes_encrypt_blocks_arm32_crypto_start_128_%=\n\t" - "bgt L_aes_encrypt_blocks_arm32_crypto_start_256_%=\n\t" - /* AES_ECB_192 */ -#ifndef NO_AES_192 - "vld1.32 {q8-q9}, [%[key]]!\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_encrypt_blocks_arm32_crypto_192_start_1_%=\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_192_start_4_%=:\n\t" - "cmp %[sz], #4\n\t" - "blt L_aes_encrypt_blocks_arm32_crypto_192_start_2_%=\n\t" - "vldm.8 %[in]!, {q12-q15}\n\t" - "aese.8 q12, q0\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q0\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q0\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q1\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q1\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q1\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q2\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q2\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q2\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q3\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q3\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q3\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q4\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q4\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q4\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q5\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q5\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q5\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q6\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q6\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q6\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q7\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q7\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q7\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q8\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q8\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q8\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q9\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q9\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q9\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aese.8 q12, q10\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q10\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q10\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "aese.8 q13, q11\n\t" - "veor.32 q13, q13, q10\n\t" - "aese.8 q14, q11\n\t" - "veor.32 q14, q14, q10\n\t" - "aese.8 q15, q11\n\t" - "veor.32 q15, q15, q10\n\t" - "sub %[key], %[key], #48\n\t" - "sub %[sz], %[sz], #4\n\t" - "vstm.8 %[out]!, {q12-q15}\n\t" - "cmp %[sz], #4\n\t" - "bge L_aes_encrypt_blocks_arm32_crypto_192_start_4_%=\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_192_start_2_%=:\n\t" - "cmp %[sz], #2\n\t" - "blt L_aes_encrypt_blocks_arm32_crypto_192_start_1_%=\n\t" - "vld1.8 {q12-q13}, [%[in]]!\n\t" - "aese.8 q12, q0\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q0\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q1\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q1\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q2\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q2\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q3\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q3\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q4\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q4\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q5\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q5\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q6\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q6\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q7\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q7\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q8\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q8\n\t" - "aesmc.8 q13, q13\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q9\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q9\n\t" - "aesmc.8 q13, q13\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aese.8 q12, q10\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q10\n\t" - "aesmc.8 q13, q13\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "aese.8 q13, q11\n\t" - "veor.32 q13, q13, q10\n\t" - "sub %[key], %[key], #48\n\t" - "sub %[sz], %[sz], #2\n\t" - "vst1.8 {q12-q13}, [%[out]]!\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_192_start_1_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_encrypt_blocks_arm32_crypto_192_done_%=\n\t" - "vld1.8 {q12}, [%[in]]!\n\t" - "aese.8 q12, q0\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q1\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q2\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q3\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q4\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q5\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q6\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q7\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q8\n\t" - "aesmc.8 q12, q12\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q9\n\t" - "aesmc.8 q12, q12\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aese.8 q12, q10\n\t" - "aesmc.8 q12, q12\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "sub %[key], %[key], #48\n\t" - "vst1.8 {q12}, [%[out]]!\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_192_done_%=:\n\t" -#endif /* !NO_AES_192 */ - "b L_aes_encrypt_blocks_arm32_crypto_done_%=\n\t" - /* AES_ECB_256 */ - "\n" - "L_aes_encrypt_blocks_arm32_crypto_start_256_%=:\n\t" -#ifndef NO_AES_256 - "vld1.32 {q8-q9}, [%[key]]!\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_encrypt_blocks_arm32_crypto_256_start_1_%=\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_256_start_4_%=:\n\t" - "cmp %[sz], #4\n\t" - "blt L_aes_encrypt_blocks_arm32_crypto_256_start_2_%=\n\t" - "vldm.8 %[in]!, {q12-q15}\n\t" - "aese.8 q12, q0\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q0\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q0\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q1\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q1\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q1\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q2\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q2\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q2\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q3\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q3\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q3\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q4\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q4\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q4\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q5\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q5\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q5\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q6\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q6\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q6\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q7\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q7\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q7\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q8\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q8\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q8\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q9\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q9\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q9\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aese.8 q12, q10\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q10\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q10\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q11\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q11\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q11\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q11\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aese.8 q12, q10\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q10\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q10\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "aese.8 q13, q11\n\t" - "veor.32 q13, q13, q10\n\t" - "aese.8 q14, q11\n\t" - "veor.32 q14, q14, q10\n\t" - "aese.8 q15, q11\n\t" - "veor.32 q15, q15, q10\n\t" - "sub %[key], %[key], #0x50\n\t" - "sub %[sz], %[sz], #4\n\t" - "vstm.8 %[out]!, {q12-q15}\n\t" - "cmp %[sz], #4\n\t" - "bge L_aes_encrypt_blocks_arm32_crypto_256_start_4_%=\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_256_start_2_%=:\n\t" - "cmp %[sz], #2\n\t" - "blt L_aes_encrypt_blocks_arm32_crypto_256_start_1_%=\n\t" - "vld1.8 {q12-q13}, [%[in]]!\n\t" - "aese.8 q12, q0\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q0\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q1\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q1\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q2\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q2\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q3\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q3\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q4\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q4\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q5\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q5\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q6\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q6\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q7\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q7\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q8\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q8\n\t" - "aesmc.8 q13, q13\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q9\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q9\n\t" - "aesmc.8 q13, q13\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aese.8 q12, q10\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q10\n\t" - "aesmc.8 q13, q13\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q11\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q11\n\t" - "aesmc.8 q13, q13\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aese.8 q12, q10\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q10\n\t" - "aesmc.8 q13, q13\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "aese.8 q13, q11\n\t" - "veor.32 q13, q13, q10\n\t" - "sub %[key], %[key], #0x50\n\t" - "sub %[sz], %[sz], #2\n\t" - "vst1.8 {q12-q13}, [%[out]]!\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_256_start_1_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_encrypt_blocks_arm32_crypto_256_done_%=\n\t" - "vld1.8 {q12}, [%[in]]!\n\t" - "aese.8 q12, q0\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q1\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q2\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q3\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q4\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q5\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q6\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q7\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q8\n\t" - "aesmc.8 q12, q12\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q9\n\t" - "aesmc.8 q12, q12\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aese.8 q12, q10\n\t" - "aesmc.8 q12, q12\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q11\n\t" - "aesmc.8 q12, q12\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aese.8 q12, q10\n\t" - "aesmc.8 q12, q12\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aese.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "sub %[key], %[key], #0x50\n\t" - "vst1.8 {q12}, [%[out]]!\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_256_done_%=:\n\t" -#endif /* !NO_AES_256 */ - "b L_aes_encrypt_blocks_arm32_crypto_done_%=\n\t" - /* AES_ECB_128 */ - "\n" - "L_aes_encrypt_blocks_arm32_crypto_start_128_%=:\n\t" -#ifndef NO_AES_128 - "vldm.32 %[key]!, {q8-q10}\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_encrypt_blocks_arm32_crypto_128_start_1_%=\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_128_start_4_%=:\n\t" - "cmp %[sz], #4\n\t" - "blt L_aes_encrypt_blocks_arm32_crypto_128_start_2_%=\n\t" - "vldm.8 %[in]!, {q12-q15}\n\t" - "aese.8 q12, q0\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q0\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q0\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q1\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q1\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q1\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q2\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q2\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q2\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q3\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q3\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q3\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q4\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q4\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q4\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q5\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q5\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q5\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q6\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q6\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q6\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q7\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q7\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q7\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q8\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q8\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q14, q8\n\t" - "aesmc.8 q14, q14\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q12, q9\n\t" - "veor.32 q12, q12, q10\n\t" - "aese.8 q13, q9\n\t" - "veor.32 q13, q13, q10\n\t" - "aese.8 q14, q9\n\t" - "veor.32 q14, q14, q10\n\t" - "aese.8 q15, q9\n\t" - "veor.32 q15, q15, q10\n\t" - "sub %[sz], %[sz], #4\n\t" - "vstm.8 %[out]!, {q12-q15}\n\t" - "cmp %[sz], #4\n\t" - "bge L_aes_encrypt_blocks_arm32_crypto_128_start_4_%=\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_128_start_2_%=:\n\t" - "cmp %[sz], #2\n\t" - "blt L_aes_encrypt_blocks_arm32_crypto_128_start_1_%=\n\t" - "vld1.8 {q12-q13}, [%[in]]!\n\t" - "aese.8 q12, q0\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q0\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q1\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q1\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q2\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q2\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q3\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q3\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q4\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q4\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q5\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q5\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q6\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q6\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q7\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q7\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q8\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q13, q8\n\t" - "aesmc.8 q13, q13\n\t" - "aese.8 q12, q9\n\t" - "veor.32 q12, q12, q10\n\t" - "aese.8 q13, q9\n\t" - "veor.32 q13, q13, q10\n\t" - "sub %[sz], %[sz], #2\n\t" - "vst1.8 {q12-q13}, [%[out]]!\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_128_start_1_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_encrypt_blocks_arm32_crypto_128_done_%=\n\t" - "vld1.8 {q12}, [%[in]]!\n\t" - "aese.8 q12, q0\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q1\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q2\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q3\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q4\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q5\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q6\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q7\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q8\n\t" - "aesmc.8 q12, q12\n\t" - "aese.8 q12, q9\n\t" - "veor.32 q12, q12, q10\n\t" - "vst1.8 {q12}, [%[out]]!\n\t" - "\n" - "L_aes_encrypt_blocks_arm32_crypto_128_done_%=:\n\t" -#endif /* !NO_AES_128 */ - "\n" - "L_aes_encrypt_blocks_arm32_crypto_done_%=:\n\t" - "pop {%[nr]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), - [nr] "+r" (nr) - : -#else - : - : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [key] "r" (key), - [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", - "q9", "q10", "q11", "q12", "q13", "q14", "q15" - ); -} - -#ifdef HAVE_AES_DECRYPT -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_decrypt_blocks_AARCH32(const byte* in_p, - byte* out_p, word32 sz_p, byte* key_p, int nr_p) -#else -WC_OMIT_FRAME_POINTER void AES_decrypt_blocks_AARCH32(const byte* in, byte* out, - word32 sz, byte* key, int nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* in __asm__ ("r0") = (const byte*)in_p; - register byte* out __asm__ ("r1") = (byte*)out_p; - register word32 sz __asm__ ("r2") = (word32)sz_p; - register byte* key __asm__ ("r3") = (byte*)key_p; - register int nr __asm__ ("r12") = (int)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[nr]}\n\t" - "ldr r12, [sp]\n\t" - "vldm.32 %[key]!, {q0-q7}\n\t" - "lsr %[sz], %[sz], #4\n\t" - "cmp r12, #12\n\t" - "blt L_aes_decrypt_blocks_arm32_crypto_start_128_%=\n\t" - "bgt L_aes_decrypt_blocks_arm32_crypto_start_256_%=\n\t" - /* AES_ECB_192 */ -#ifndef NO_AES_192 - "vld1.32 {q8-q9}, [%[key]]!\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_decrypt_blocks_arm32_crypto_192_start_1_%=\n\t" - "cmp %[sz], #4\n\t" - "blt L_aes_decrypt_blocks_arm32_crypto_192_start_2_%=\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_192_start_4_%=:\n\t" - "vldm.8 %[in]!, {q12-q15}\n\t" - "aesd.8 q12, q0\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q0\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q0\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q0\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q1\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q1\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q1\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q1\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q2\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q2\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q2\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q2\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q3\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q3\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q3\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q3\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q4\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q4\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q4\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q4\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q5\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q5\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q5\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q5\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q6\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q6\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q6\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q6\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q7\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q7\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q7\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q7\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q8\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q8\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q8\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q8\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q9\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q9\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q9\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q9\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aesd.8 q12, q10\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q10\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q10\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q10\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "aesd.8 q13, q11\n\t" - "veor.32 q13, q13, q10\n\t" - "aesd.8 q14, q11\n\t" - "veor.32 q14, q14, q10\n\t" - "aesd.8 q15, q11\n\t" - "veor.32 q15, q15, q10\n\t" - "sub %[key], %[key], #48\n\t" - "sub %[sz], %[sz], #4\n\t" - "vstm.8 %[out]!, {q12-q15}\n\t" - "cmp %[sz], #4\n\t" - "bge L_aes_decrypt_blocks_arm32_crypto_192_start_4_%=\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_192_start_2_%=:\n\t" - "cmp %[sz], #2\n\t" - "blt L_aes_decrypt_blocks_arm32_crypto_192_start_1_%=\n\t" - "vld1.8 {q12-q13}, [%[in]]!\n\t" - "aesd.8 q12, q0\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q0\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q1\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q1\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q2\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q2\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q3\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q3\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q4\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q4\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q5\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q5\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q6\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q6\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q7\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q7\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q8\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q8\n\t" - "aesimc.8 q13, q13\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q9\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q9\n\t" - "aesimc.8 q13, q13\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aesd.8 q12, q10\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q10\n\t" - "aesimc.8 q13, q13\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "aesd.8 q13, q11\n\t" - "veor.32 q13, q13, q10\n\t" - "sub %[key], %[key], #48\n\t" - "sub %[sz], %[sz], #2\n\t" - "vst1.8 {q12-q13}, [%[out]]!\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_192_start_1_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_decrypt_blocks_arm32_crypto_192_done_%=\n\t" - "vld1.8 {q12}, [%[in]]!\n\t" - "aesd.8 q12, q0\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q1\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q2\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q3\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q4\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q5\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q6\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q7\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q8\n\t" - "aesimc.8 q12, q12\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q9\n\t" - "aesimc.8 q12, q12\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aesd.8 q12, q10\n\t" - "aesimc.8 q12, q12\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "sub %[key], %[key], #48\n\t" - "vst1.8 {q12}, [%[out]]!\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_192_done_%=:\n\t" -#endif /* !NO_AES_192 */ - "b L_aes_decrypt_blocks_arm32_crypto_done_%=\n\t" - /* AES_ECB_256 */ - "\n" - "L_aes_decrypt_blocks_arm32_crypto_start_256_%=:\n\t" -#ifndef NO_AES_256 - "vld1.32 {q8-q9}, [%[key]]!\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_decrypt_blocks_arm32_crypto_256_start_1_%=\n\t" - "cmp %[sz], #4\n\t" - "blt L_aes_decrypt_blocks_arm32_crypto_256_start_2_%=\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_256_start_4_%=:\n\t" - "vldm.8 %[in]!, {q12-q15}\n\t" - "aesd.8 q12, q0\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q0\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q0\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q0\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q1\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q1\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q1\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q1\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q2\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q2\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q2\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q2\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q3\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q3\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q3\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q3\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q4\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q4\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q4\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q4\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q5\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q5\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q5\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q5\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q6\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q6\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q6\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q6\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q7\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q7\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q7\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q7\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q8\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q8\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q8\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q8\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q9\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q9\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q9\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q9\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aesd.8 q12, q10\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q10\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q10\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q10\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q11\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q11\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q11\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q11\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aesd.8 q12, q10\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q10\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q10\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q10\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "aesd.8 q13, q11\n\t" - "veor.32 q13, q13, q10\n\t" - "aesd.8 q14, q11\n\t" - "veor.32 q14, q14, q10\n\t" - "aesd.8 q15, q11\n\t" - "veor.32 q15, q15, q10\n\t" - "sub %[key], %[key], #0x50\n\t" - "sub %[sz], %[sz], #4\n\t" - "vstm.8 %[out]!, {q12-q15}\n\t" - "cmp %[sz], #4\n\t" - "bge L_aes_decrypt_blocks_arm32_crypto_256_start_4_%=\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_256_start_2_%=:\n\t" - "cmp %[sz], #2\n\t" - "blt L_aes_decrypt_blocks_arm32_crypto_256_start_1_%=\n\t" - "vld1.8 {q12-q13}, [%[in]]!\n\t" - "aesd.8 q12, q0\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q0\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q1\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q1\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q2\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q2\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q3\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q3\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q4\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q4\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q5\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q5\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q6\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q6\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q7\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q7\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q8\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q8\n\t" - "aesimc.8 q13, q13\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q9\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q9\n\t" - "aesimc.8 q13, q13\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aesd.8 q12, q10\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q10\n\t" - "aesimc.8 q13, q13\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q11\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q11\n\t" - "aesimc.8 q13, q13\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aesd.8 q12, q10\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q10\n\t" - "aesimc.8 q13, q13\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "aesd.8 q13, q11\n\t" - "veor.32 q13, q13, q10\n\t" - "sub %[key], %[key], #0x50\n\t" - "sub %[sz], %[sz], #2\n\t" - "vst1.8 {q12-q13}, [%[out]]!\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_256_start_1_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_decrypt_blocks_arm32_crypto_256_done_%=\n\t" - "vld1.8 {q12}, [%[in]]!\n\t" - "aesd.8 q12, q0\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q1\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q2\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q3\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q4\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q5\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q6\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q7\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q8\n\t" - "aesimc.8 q12, q12\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q9\n\t" - "aesimc.8 q12, q12\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aesd.8 q12, q10\n\t" - "aesimc.8 q12, q12\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q11\n\t" - "aesimc.8 q12, q12\n\t" - "vld1.32 {q11}, [%[key]]!\n\t" - "aesd.8 q12, q10\n\t" - "aesimc.8 q12, q12\n\t" - "vld1.32 {q10}, [%[key]]!\n\t" - "aesd.8 q12, q11\n\t" - "veor.32 q12, q12, q10\n\t" - "sub %[key], %[key], #0x50\n\t" - "vst1.8 {q12}, [%[out]]!\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_256_done_%=:\n\t" -#endif /* !NO_AES_256 */ - "b L_aes_decrypt_blocks_arm32_crypto_done_%=\n\t" - /* AES_ECB_128 */ - "\n" - "L_aes_decrypt_blocks_arm32_crypto_start_128_%=:\n\t" -#ifndef NO_AES_128 - "vldm.32 %[key]!, {q8-q10}\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_decrypt_blocks_arm32_crypto_128_start_1_%=\n\t" - "cmp %[sz], #4\n\t" - "blt L_aes_decrypt_blocks_arm32_crypto_128_start_2_%=\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_128_start_4_%=:\n\t" - "vldm.8 %[in]!, {q12-q15}\n\t" - "aesd.8 q12, q0\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q0\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q0\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q0\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q1\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q1\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q1\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q1\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q2\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q2\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q2\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q2\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q3\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q3\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q3\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q3\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q4\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q4\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q4\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q4\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q5\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q5\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q5\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q5\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q6\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q6\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q6\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q6\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q7\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q7\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q7\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q7\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q8\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q8\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q14, q8\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q8\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q12, q9\n\t" - "veor.32 q12, q12, q10\n\t" - "aesd.8 q13, q9\n\t" - "veor.32 q13, q13, q10\n\t" - "aesd.8 q14, q9\n\t" - "veor.32 q14, q14, q10\n\t" - "aesd.8 q15, q9\n\t" - "veor.32 q15, q15, q10\n\t" - "sub %[sz], %[sz], #4\n\t" - "vstm.8 %[out]!, {q12-q15}\n\t" - "cmp %[sz], #4\n\t" - "bge L_aes_decrypt_blocks_arm32_crypto_128_start_4_%=\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_128_start_2_%=:\n\t" - "cmp %[sz], #2\n\t" - "blt L_aes_decrypt_blocks_arm32_crypto_128_start_1_%=\n\t" - "vld1.8 {q12-q13}, [%[in]]!\n\t" - "aesd.8 q12, q0\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q0\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q1\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q1\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q2\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q2\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q3\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q3\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q4\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q4\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q5\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q5\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q6\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q6\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q7\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q7\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q8\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q13, q8\n\t" - "aesimc.8 q13, q13\n\t" - "aesd.8 q12, q9\n\t" - "veor.32 q12, q12, q10\n\t" - "aesd.8 q13, q9\n\t" - "veor.32 q13, q13, q10\n\t" - "sub %[sz], %[sz], #2\n\t" - "vst1.8 {q12-q13}, [%[out]]!\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_128_start_1_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_decrypt_blocks_arm32_crypto_128_done_%=\n\t" - "vld1.8 {q12}, [%[in]]!\n\t" - "aesd.8 q12, q0\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q1\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q2\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q3\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q4\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q5\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q6\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q7\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q8\n\t" - "aesimc.8 q12, q12\n\t" - "aesd.8 q12, q9\n\t" - "veor.32 q12, q12, q10\n\t" - "vst1.8 {q12}, [%[out]]!\n\t" - "\n" - "L_aes_decrypt_blocks_arm32_crypto_128_done_%=:\n\t" -#endif /* !NO_AES_128 */ - "\n" - "L_aes_decrypt_blocks_arm32_crypto_done_%=:\n\t" - "pop {%[nr]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), - [nr] "+r" (nr) - : -#else - : - : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [key] "r" (key), - [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", - "q9", "q10", "q11", "q12", "q13", "q14", "q15" - ); -} - -#endif /* HAVE_AES_DECRYPT */ -#endif /* HAVE_AES_ECB */ -#ifdef HAVE_AES_CBC -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_CBC_encrypt_AARCH32(const byte* in_p, - byte* out_p, word32 sz_p, byte* reg_p, byte* key_p, int nr_p) -#else -WC_OMIT_FRAME_POINTER void AES_CBC_encrypt_AARCH32(const byte* in, byte* out, - word32 sz, byte* reg, byte* key, int nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* in __asm__ ("r0") = (const byte*)in_p; - register byte* out __asm__ ("r1") = (byte*)out_p; - register word32 sz __asm__ ("r2") = (word32)sz_p; - register byte* reg __asm__ ("r3") = (byte*)reg_p; - register byte* key __asm__ ("r12") = (byte*)key_p; - register int nr __asm__ ("lr") = (int)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[key], %[nr]}\n\t" - "ldr r12, [sp]\n\t" - "ldr lr, [sp, #4]\n\t" - "vldm.32 r12!, {q0-q7}\n\t" - "vld1.32 {q15}, [%[reg]]\n\t" - "subs lr, lr, #12\n\t" - "lsr %[sz], %[sz], #4\n\t" - "blt L_aes_cbc_encrypt_arm32_crypto_start_128_%=\n\t" - "bgt L_aes_cbc_encrypt_arm32_crypto_start_256_%=\n\t" - /* AES_CBC_192 */ -#ifndef NO_AES_192 - "vld1.8 {q14}, [%[in]]!\n\t" - "vldm.32 r12!, {q8-q12}\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_cbc_encrypt_arm32_crypto_192_start_1_%=\n\t" - "cmp %[sz], #4\n\t" - "blt L_aes_cbc_encrypt_arm32_crypto_192_start_2_%=\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_192_start_4_%=:\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q11\n\t" - "veor.32 q15, q15, q12\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q11\n\t" - "veor.32 q15, q15, q12\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q11\n\t" - "veor.32 q15, q15, q12\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "subs %[sz], %[sz], #4\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q11\n\t" - "veor.32 q15, q15, q12\n\t" - "beq L_aes_cbc_encrypt_arm32_crypto_192_done_%=\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "cmp %[sz], #4\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "bge L_aes_cbc_encrypt_arm32_crypto_192_start_4_%=\n\t" - "cmp %[sz], #2\n\t" - "blt L_aes_cbc_encrypt_arm32_crypto_192_start_1_%=\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_192_start_2_%=:\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q11\n\t" - "veor.32 q15, q15, q12\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "subs %[sz], %[sz], #2\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q11\n\t" - "veor.32 q15, q15, q12\n\t" - "beq L_aes_cbc_encrypt_arm32_crypto_192_done_%=\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_192_start_1_%=:\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q11\n\t" - "veor.32 q15, q15, q12\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_192_done_%=:\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" -#endif /* !NO_AES_192 */ - "b L_aes_cbc_encrypt_arm32_crypto_done_%=\n\t" - /* AES_CBC_256 */ - "\n" - "L_aes_cbc_encrypt_arm32_crypto_start_256_%=:\n\t" -#ifndef NO_AES_256 - "vld1.8 {q14}, [%[in]]!\n\t" - "vldm.32 r12!, {q8-q11}\n\t" - "add r12, r12, #16\n\t" - "vld1.32 {q12-q13}, [r12]\n\t" - "sub r12, r12, #16\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_cbc_encrypt_arm32_crypto_256_start_1_%=\n\t" - "cmp %[sz], #4\n\t" - "blt L_aes_cbc_encrypt_arm32_crypto_256_start_2_%=\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_256_start_4_%=:\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q14}, [r12]\n\t" - "aese.8 q15, q11\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q14\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q12\n\t" - "veor.32 q15, q15, q13\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q14}, [r12]\n\t" - "aese.8 q15, q11\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q14\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q12\n\t" - "veor.32 q15, q15, q13\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q14}, [r12]\n\t" - "aese.8 q15, q11\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q14\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q12\n\t" - "veor.32 q15, q15, q13\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q14}, [r12]\n\t" - "aese.8 q15, q11\n\t" - "aesmc.8 q15, q15\n\t" - "subs %[sz], %[sz], #4\n\t" - "aese.8 q15, q14\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q12\n\t" - "veor.32 q15, q15, q13\n\t" - "beq L_aes_cbc_encrypt_arm32_crypto_256_done_%=\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "cmp %[sz], #4\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "bge L_aes_cbc_encrypt_arm32_crypto_256_start_4_%=\n\t" - "cmp %[sz], #2\n\t" - "blt L_aes_cbc_encrypt_arm32_crypto_256_start_1_%=\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_256_start_2_%=:\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q14}, [r12]\n\t" - "aese.8 q15, q11\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q14\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q12\n\t" - "veor.32 q15, q15, q13\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q14}, [r12]\n\t" - "aese.8 q15, q11\n\t" - "aesmc.8 q15, q15\n\t" - "subs %[sz], %[sz], #2\n\t" - "aese.8 q15, q14\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q12\n\t" - "veor.32 q15, q15, q13\n\t" - "beq L_aes_cbc_encrypt_arm32_crypto_256_done_%=\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_256_start_1_%=:\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q10\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.32 {q14}, [r12]\n\t" - "aese.8 q15, q11\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q14\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q12\n\t" - "veor.32 q15, q15, q13\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_256_done_%=:\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" -#endif /* !NO_AES_256 */ - "b L_aes_cbc_encrypt_arm32_crypto_done_%=\n\t" - /* AES_CBC_128 */ - "\n" - "L_aes_cbc_encrypt_arm32_crypto_start_128_%=:\n\t" -#ifndef NO_AES_128 - "vld1.8 {q14}, [%[in]]!\n\t" - "vldm.32 r12!, {q8-q10}\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_cbc_encrypt_arm32_crypto_128_start_1_%=\n\t" - "cmp %[sz], #4\n\t" - "blt L_aes_cbc_encrypt_arm32_crypto_128_start_2_%=\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_128_start_4_%=:\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q9\n\t" - "veor.32 q15, q15, q10\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q9\n\t" - "veor.32 q15, q15, q10\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q9\n\t" - "veor.32 q15, q15, q10\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "subs %[sz], %[sz], #4\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "veor.32 q15, q15, q10\n\t" - "beq L_aes_cbc_encrypt_arm32_crypto_128_done_%=\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "cmp %[sz], #4\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "bge L_aes_cbc_encrypt_arm32_crypto_128_start_4_%=\n\t" - "cmp %[sz], #2\n\t" - "blt L_aes_cbc_encrypt_arm32_crypto_128_start_1_%=\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_128_start_2_%=:\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q15, q9\n\t" - "veor.32 q15, q15, q10\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "subs %[sz], %[sz], #2\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "veor.32 q15, q15, q10\n\t" - "beq L_aes_cbc_encrypt_arm32_crypto_128_done_%=\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_128_start_1_%=:\n\t" - "veor.32 q15, q15, q14\n\t" - "aese.8 q15, q0\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q1\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q2\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q3\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q4\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q5\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q6\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q7\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q8\n\t" - "aesmc.8 q15, q15\n\t" - "aese.8 q15, q9\n\t" - "veor.32 q15, q15, q10\n\t" - "\n" - "L_aes_cbc_encrypt_arm32_crypto_128_done_%=:\n\t" - "vst1.8 {q15}, [%[out]]!\n\t" -#endif /* !NO_AES_128 */ - "\n" - "L_aes_cbc_encrypt_arm32_crypto_done_%=:\n\t" - "vst1.32 {q15}, [%[reg]]\n\t" - "pop {%[key], %[nr]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [reg] "+r" (reg), - [key] "+r" (key), [nr] "+r" (nr) - : -#else - : - : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [reg] "r" (reg), - [key] "r" (key), [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", - "q9", "q10", "q11", "q12", "q13", "q14", "q15" - ); -} - -#ifdef HAVE_AES_DECRYPT -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_CBC_decrypt_AARCH32(const byte* in_p, - byte* out_p, word32 sz_p, byte* reg_p, byte* key_p, int nr_p) -#else -WC_OMIT_FRAME_POINTER void AES_CBC_decrypt_AARCH32(const byte* in, byte* out, - word32 sz, byte* reg, byte* key, int nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* in __asm__ ("r0") = (const byte*)in_p; - register byte* out __asm__ ("r1") = (byte*)out_p; - register word32 sz __asm__ ("r2") = (word32)sz_p; - register byte* reg __asm__ ("r3") = (byte*)reg_p; - register byte* key __asm__ ("r12") = (byte*)key_p; - register int nr __asm__ ("lr") = (int)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[key], %[nr]}\n\t" - "ldr r12, [sp]\n\t" - "ldr lr, [sp, #4]\n\t" - "vldm.32 r12!, {q0-q7}\n\t" - "vld1.32 {q13}, [%[reg]]\n\t" - "lsr %[sz], %[sz], #4\n\t" - "cmp lr, #12\n\t" - "blt L_aes_cbc_decrypt_blocks_arm32_crypto_start_128_%=\n\t" - "bgt L_aes_cbc_decrypt_blocks_arm32_crypto_start_256_%=\n\t" - /* AES_CBC_192 */ -#ifndef NO_AES_192 - "vld1.32 {q8}, [r12]!\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_1_%=\n\t" - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_2_%=:\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "vmov q11, q13\n\t" - "vmov q12, q14\n\t" - "vmov q13, q15\n\t" - "aesd.8 q14, q0\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q0\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q1\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q1\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q2\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q2\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q3\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q3\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q4\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q4\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q5\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q5\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q6\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q6\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q7\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q7\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q9}, [r12]!\n\t" - "aesd.8 q14, q8\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q8\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q10}, [r12]!\n\t" - "aesd.8 q14, q9\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q9\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q9}, [r12]!\n\t" - "aesd.8 q14, q10\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q10\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q10}, [r12]\n\t" - "aesd.8 q14, q9\n\t" - "aesd.8 q15, q9\n\t" - "sub %[sz], %[sz], #2\n\t" - "veor.32 q14, q14, q10\n\t" - "veor.32 q15, q15, q10\n\t" - "cmp %[sz], #1\n\t" - "veor.32 q14, q14, q11\n\t" - "veor.32 q15, q15, q12\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "sub r12, r12, #48\n\t" - "blt L_aes_cbc_decrypt_blocks_arm32_crypto_192_done_%=\n\t" - "bgt L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_2_%=\n\t" - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_1_%=:\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "vmov q11, q13\n\t" - "vmov q13, q14\n\t" - "aesd.8 q14, q0\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q1\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q2\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q3\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q4\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q5\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q6\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q7\n\t" - "aesimc.8 q14, q14\n\t" - "vld1.32 {q9}, [r12]!\n\t" - "aesd.8 q14, q8\n\t" - "aesimc.8 q14, q14\n\t" - "vld1.32 {q10}, [r12]!\n\t" - "aesd.8 q14, q9\n\t" - "aesimc.8 q14, q14\n\t" - "vld1.32 {q9}, [r12]!\n\t" - "aesd.8 q14, q10\n\t" - "aesimc.8 q14, q14\n\t" - "vld1.32 {q10}, [r12]\n\t" - "aesd.8 q14, q9\n\t" - "veor.32 q14, q14, q10\n\t" - "veor.32 q14, q14, q11\n\t" - "vst1.8 {q14}, [%[out]]!\n\t" - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_192_done_%=:\n\t" -#endif /* !NO_AES_192 */ - "b L_aes_cbc_decrypt_blocks_arm32_crypto_done_%=\n\t" - /* AES_CBC_256 */ - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_start_256_%=:\n\t" -#ifndef NO_AES_256 - "vld1.32 {q8}, [r12]!\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_1_%=\n\t" - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_2_%=:\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "vmov q11, q13\n\t" - "vmov q12, q14\n\t" - "vmov q13, q15\n\t" - "aesd.8 q14, q0\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q0\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q1\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q1\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q2\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q2\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q3\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q3\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q4\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q4\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q5\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q5\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q6\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q6\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q7\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q7\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q9}, [r12]!\n\t" - "aesd.8 q14, q8\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q8\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q10}, [r12]!\n\t" - "aesd.8 q14, q9\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q9\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q9}, [r12]!\n\t" - "aesd.8 q14, q10\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q10\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q10}, [r12]!\n\t" - "aesd.8 q14, q9\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q9\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q9}, [r12]!\n\t" - "aesd.8 q14, q10\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q10\n\t" - "aesimc.8 q15, q15\n\t" - "vld1.32 {q10}, [r12]\n\t" - "aesd.8 q14, q9\n\t" - "aesd.8 q15, q9\n\t" - "sub %[sz], %[sz], #2\n\t" - "veor.32 q14, q14, q10\n\t" - "veor.32 q15, q15, q10\n\t" - "cmp %[sz], #1\n\t" - "veor.32 q14, q14, q11\n\t" - "veor.32 q15, q15, q12\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "sub r12, r12, #0x50\n\t" - "blt L_aes_cbc_decrypt_blocks_arm32_crypto_256_done_%=\n\t" - "bgt L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_2_%=\n\t" - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_1_%=:\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "vmov q11, q13\n\t" - "vmov q13, q14\n\t" - "aesd.8 q14, q0\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q1\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q2\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q3\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q4\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q5\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q6\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q7\n\t" - "aesimc.8 q14, q14\n\t" - "vld1.32 {q9}, [r12]!\n\t" - "aesd.8 q14, q8\n\t" - "aesimc.8 q14, q14\n\t" - "vld1.32 {q10}, [r12]!\n\t" - "aesd.8 q14, q9\n\t" - "aesimc.8 q14, q14\n\t" - "vld1.32 {q9}, [r12]!\n\t" - "aesd.8 q14, q10\n\t" - "aesimc.8 q14, q14\n\t" - "vld1.32 {q10}, [r12]!\n\t" - "aesd.8 q14, q9\n\t" - "aesimc.8 q14, q14\n\t" - "vld1.32 {q9}, [r12]!\n\t" - "aesd.8 q14, q10\n\t" - "aesimc.8 q14, q14\n\t" - "vld1.32 {q10}, [r12]\n\t" - "aesd.8 q14, q9\n\t" - "veor.32 q14, q14, q10\n\t" - "veor.32 q14, q14, q11\n\t" - "vst1.8 {q14}, [%[out]]!\n\t" - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_256_done_%=:\n\t" -#endif /* !NO_AES_256 */ - "b L_aes_cbc_decrypt_blocks_arm32_crypto_done_%=\n\t" - /* AES_CBC_128 */ - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_start_128_%=:\n\t" -#ifndef NO_AES_128 - "vldm.32 r12!, {q8-q10}\n\t" - "cmp %[sz], #1\n\t" - "beq L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_1_%=\n\t" - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_2_%=:\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "vmov q11, q13\n\t" - "vmov q12, q14\n\t" - "vmov q13, q15\n\t" - "aesd.8 q14, q0\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q0\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q1\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q1\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q2\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q2\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q3\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q3\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q4\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q4\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q5\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q5\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q6\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q6\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q7\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q7\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q8\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q15, q8\n\t" - "aesimc.8 q15, q15\n\t" - "aesd.8 q14, q9\n\t" - "aesd.8 q15, q9\n\t" - "sub %[sz], %[sz], #2\n\t" - "veor.32 q14, q14, q10\n\t" - "veor.32 q15, q15, q10\n\t" - "cmp %[sz], #1\n\t" - "veor.32 q14, q14, q11\n\t" - "veor.32 q15, q15, q12\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "blt L_aes_cbc_decrypt_blocks_arm32_crypto_128_done_%=\n\t" - "bgt L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_2_%=\n\t" - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_1_%=:\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "vmov q11, q13\n\t" - "vmov q13, q14\n\t" - "aesd.8 q14, q0\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q1\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q2\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q3\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q4\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q5\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q6\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q7\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q8\n\t" - "aesimc.8 q14, q14\n\t" - "aesd.8 q14, q9\n\t" - "veor.32 q14, q14, q10\n\t" - "veor.32 q14, q14, q11\n\t" - "vst1.8 {q14}, [%[out]]!\n\t" - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_128_done_%=:\n\t" -#endif /* !NO_AES_128 */ - "\n" - "L_aes_cbc_decrypt_blocks_arm32_crypto_done_%=:\n\t" - "vst1.32 {q13}, [%[reg]]\n\t" - "pop {%[key], %[nr]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [reg] "+r" (reg), - [key] "+r" (key), [nr] "+r" (nr) - : -#else - : - : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [reg] "r" (reg), - [key] "r" (key), [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", - "q9", "q10", "q11", "q12", "q13", "q14", "q15" - ); -} - -#endif /* HAVE_AES_DECRYPT */ -#endif /* HAVE_AES_CBC */ -#ifdef WOLFSSL_AES_COUNTER -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_CTR_encrypt_AARCH32(const byte* in_p, - byte* out_p, word32 sz_p, byte* reg_p, byte* key_p, byte* tmp_p, - word32* left_p, word32 nr_p) -#else -WC_OMIT_FRAME_POINTER void AES_CTR_encrypt_AARCH32(const byte* in, byte* out, - word32 sz, byte* reg, byte* key, byte* tmp, word32* left, word32 nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* in __asm__ ("r0") = (const byte*)in_p; - register byte* out __asm__ ("r1") = (byte*)out_p; - register word32 sz __asm__ ("r2") = (word32)sz_p; - register byte* reg __asm__ ("r3") = (byte*)reg_p; - register byte* key __asm__ ("r12") = (byte*)key_p; - register byte* tmp __asm__ ("lr") = (byte*)tmp_p; - register word32* left __asm__ ("r4") = (word32*)left_p; - register word32 nr __asm__ ("r5") = (word32)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[left], %[nr]}\n\t" - "push {%[key], %[tmp]}\n\t" - "vld1.32 {q0}, [%[reg]]\n\t" - "ldr r12, [sp]\n\t" - "vrev32.8 q2, q0\n\t" - "lsr r4, %[sz], #4\n\t" - "vmov r5, r6, d4\n\t" - "and %[sz], %[sz], #15\n\t" - "vmov r7, r8, d5\n\t" - "vldm.32 r12!, {q3-q10}\n\t" - "ldr lr, [sp, #12]\n\t" - "cmp lr, #12\n\t" - "blt L_aes_ctr_encrypt_arm32_crypto_start_128_%=\n\t" - "bgt L_aes_ctr_encrypt_arm32_crypto_start_256_%=\n\t" - /* AES_CTR_192 */ -#ifndef NO_AES_192 - "vldm.32 r12!, {q11-q13}\n\t" - "mov lr, #1\n\t" - "cmp r4, #1\n\t" - "blt L_aes_ctr_encrypt_arm32_crypto_192_done_%=\n\t" - "beq L_aes_ctr_encrypt_arm32_crypto_192_start_1_%=\n\t" - "adds r8, r8, #1\n\t" - "adcs r7, r7, #0\n\t" - "adcs r6, r6, #0\n\t" - "adc r5, r5, #0\n\t" - "vmov d3, r7, r8\n\t" - "vmov d2, r5, r6\n\t" - "vrev32.8 q1, q1\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_192_start_2_%=:\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "adds r8, r8, #1\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "adcs r7, r7, #0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "adcs r6, r6, #0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "adc r5, r5, #0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q0, q11\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q11\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q0, q12\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q12\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "vmov d5, r7, r8\n\t" - "aese.8 q0, q13\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q13\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q15}, [r12]\n\t" - "vmov d4, r5, r6\n\t" - "aese.8 q0, q14\n\t" - "aese.8 q1, q14\n\t" - "sub r4, r4, #2\n\t" - "veor.32 q0, q0, q15\n\t" - "veor.32 q1, q1, q15\n\t" - "adds r8, r8, #1\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "adcs r7, r7, #0\n\t" - "sub r12, r12, #16\n\t" - "veor.32 q14, q14, q0\n\t" - "veor.32 q15, q15, q1\n\t" - "adcs r6, r6, #0\n\t" - "vrev32.8 q0, q2\n\t" - "adc r5, r5, #0\n\t" - "vmov d2, r5, r6\n\t" - "vmov d3, r7, r8\n\t" - "cmp r4, #1\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "vrev32.8 q1, q1\n\t" - "bgt L_aes_ctr_encrypt_arm32_crypto_192_start_2_%=\n\t" - "mov lr, #0\n\t" - "blt L_aes_ctr_encrypt_arm32_crypto_192_done_%=\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_192_start_1_%=:\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "adds r8, r8, lr\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r7, r7, #0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r6, r6, #0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "adc r5, r5, #0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d5, r7, r8\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d4, r5, r6\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q11\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q12\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aese.8 q0, q13\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q15}, [r12]\n\t" - "aese.8 q0, q14\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "veor.32 q0, q0, q15\n\t" - "veor.32 q14, q14, q0\n\t" - "sub r12, r12, #16\n\t" - "vst1.8 {q14}, [%[out]]!\n\t" - "mov lr, #1\n\t" - "vrev32.8 q0, q2\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_192_done_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_ctr_encrypt_arm32_crypto_192_partial_done_%=\n\t" - "ldr r4, [sp, #8]\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "adds r8, r8, lr\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r7, r7, #0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r6, r6, #0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "adc r5, r5, #0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d5, r7, r8\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d4, r5, r6\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q11\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q12\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aese.8 q0, q13\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q15}, [r12]\n\t" - "ldr lr, [sp, #4]\n\t" - "aese.8 q0, q14\n\t" - "veor.32 q0, q0, q15\n\t" - "vst1.32 {q0}, [lr]\n\t" - "vmov q0, q2\n\t" - "mov r5, #16\n\t" - "sub r5, r5, %[sz]\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_192_start_byte_%=:\n\t" - "ldrb r7, [lr], #1\n\t" - "ldrb r8, [%[in]], #1\n\t" - "eor r7, r7, r8\n\t" - "subs %[sz], %[sz], #1\n\t" - "strb r7, [%[out]], #1\n\t" - "bgt L_aes_ctr_encrypt_arm32_crypto_192_start_byte_%=\n\t" - "vrev32.8 q0, q2\n\t" - "str r5, [r4]\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_192_partial_done_%=:\n\t" -#endif /* !NO_AES_192 */ - "b L_aes_ctr_encrypt_arm32_crypto_done_%=\n\t" - /* AES_CTR_256 */ - "\n" - "L_aes_ctr_encrypt_arm32_crypto_start_256_%=:\n\t" -#ifndef NO_AES_256 - "vldm.32 r12!, {q11-q13}\n\t" - "mov lr, #1\n\t" - "cmp r4, #1\n\t" - "blt L_aes_ctr_encrypt_arm32_crypto_256_done_%=\n\t" - "beq L_aes_ctr_encrypt_arm32_crypto_256_start_1_%=\n\t" - "adds r8, r8, #1\n\t" - "adcs r7, r7, #0\n\t" - "adcs r6, r6, #0\n\t" - "adc r5, r5, #0\n\t" - "vmov d3, r7, r8\n\t" - "vmov d2, r5, r6\n\t" - "vrev32.8 q1, q1\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_256_start_2_%=:\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "adds r8, r8, #1\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "adcs r7, r7, #0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "adcs r6, r6, #0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "adc r5, r5, #0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q0, q11\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q11\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q0, q12\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q12\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "vmov d5, r7, r8\n\t" - "aese.8 q0, q13\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q13\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q15}, [r12]!\n\t" - "vmov d4, r5, r6\n\t" - "aese.8 q0, q14\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q14\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "adds r8, r8, #1\n\t" - "aese.8 q0, q15\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q15\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q15}, [r12]\n\t" - "adcs r7, r7, #0\n\t" - "aese.8 q0, q14\n\t" - "aese.8 q1, q14\n\t" - "sub r4, r4, #2\n\t" - "veor.32 q0, q0, q15\n\t" - "veor.32 q1, q1, q15\n\t" - "adcs r6, r6, #0\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "sub r12, r12, #48\n\t" - "veor.32 q14, q14, q0\n\t" - "veor.32 q15, q15, q1\n\t" - "adc r5, r5, #0\n\t" - "vrev32.8 q0, q2\n\t" - "vmov d2, r5, r6\n\t" - "vmov d3, r7, r8\n\t" - "cmp r4, #1\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "vrev32.8 q1, q1\n\t" - "bgt L_aes_ctr_encrypt_arm32_crypto_256_start_2_%=\n\t" - "mov lr, #0\n\t" - "blt L_aes_ctr_encrypt_arm32_crypto_256_done_%=\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_256_start_1_%=:\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "adds r8, r8, lr\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r7, r7, #0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r6, r6, #0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "adc r5, r5, #0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d5, r7, r8\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d4, r5, r6\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q11\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q12\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aese.8 q0, q13\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q15}, [r12]!\n\t" - "aese.8 q0, q14\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aese.8 q0, q15\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q15}, [r12]\n\t" - "aese.8 q0, q14\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "veor.32 q0, q0, q15\n\t" - "veor.32 q14, q14, q0\n\t" - "sub r12, r12, #48\n\t" - "vst1.8 {q14}, [%[out]]!\n\t" - "mov lr, #1\n\t" - "vrev32.8 q0, q2\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_256_done_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_ctr_encrypt_arm32_crypto_256_partial_done_%=\n\t" - "ldr r4, [sp, #8]\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "adds r8, r8, lr\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r7, r7, #0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r6, r6, #0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "adc r5, r5, #0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d5, r7, r8\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d4, r5, r6\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q11\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q12\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aese.8 q0, q13\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q15}, [r12]!\n\t" - "aese.8 q0, q14\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aese.8 q0, q15\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q15}, [r12]\n\t" - "ldr lr, [sp, #4]\n\t" - "aese.8 q0, q14\n\t" - "veor.32 q0, q0, q15\n\t" - "vst1.32 {q0}, [lr]\n\t" - "vmov q0, q2\n\t" - "mov r5, #16\n\t" - "sub r5, r5, %[sz]\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_256_start_byte_%=:\n\t" - "ldrb r7, [lr], #1\n\t" - "ldrb r8, [%[in]], #1\n\t" - "eor r7, r7, r8\n\t" - "subs %[sz], %[sz], #1\n\t" - "strb r7, [%[out]], #1\n\t" - "bgt L_aes_ctr_encrypt_arm32_crypto_256_start_byte_%=\n\t" - "vrev32.8 q0, q2\n\t" - "str r5, [r4]\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_256_partial_done_%=:\n\t" -#endif /* !NO_AES_256 */ - "b L_aes_ctr_encrypt_arm32_crypto_done_%=\n\t" - /* AES_CTR_128 */ - "\n" - "L_aes_ctr_encrypt_arm32_crypto_start_128_%=:\n\t" -#ifndef NO_AES_128 - "vldm.32 r12!, {q11-q13}\n\t" - "mov lr, #1\n\t" - "cmp r4, #1\n\t" - "blt L_aes_ctr_encrypt_arm32_crypto_128_done_%=\n\t" - "beq L_aes_ctr_encrypt_arm32_crypto_128_start_1_%=\n\t" - "adds r8, r8, #1\n\t" - "adcs r7, r7, #0\n\t" - "adcs r6, r6, #0\n\t" - "adc r5, r5, #0\n\t" - "vmov d3, r7, r8\n\t" - "vmov d2, r5, r6\n\t" - "vrev32.8 q1, q1\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_128_start_2_%=:\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "adds r8, r8, #1\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "adcs r7, r7, #0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "adcs r6, r6, #0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "adc r5, r5, #0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "vmov d5, r7, r8\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "vmov d4, r5, r6\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "adds r8, r8, #1\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "adcs r7, r7, #0\n\t" - "aese.8 q0, q11\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q1, q11\n\t" - "aesmc.8 q1, q1\n\t" - "adcs r6, r6, #0\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "adc r5, r5, #0\n\t" - "aese.8 q0, q12\n\t" - "aese.8 q1, q12\n\t" - "sub r4, r4, #2\n\t" - "veor.32 q14, q14, q13\n\t" - "veor.32 q15, q15, q13\n\t" - "veor.32 q14, q14, q0\n\t" - "veor.32 q15, q15, q1\n\t" - "vrev32.8 q0, q2\n\t" - "vmov d2, r5, r6\n\t" - "vmov d3, r7, r8\n\t" - "cmp r4, #1\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "vrev32.8 q1, q1\n\t" - "bgt L_aes_ctr_encrypt_arm32_crypto_128_start_2_%=\n\t" - "mov lr, #0\n\t" - "blt L_aes_ctr_encrypt_arm32_crypto_128_done_%=\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_128_start_1_%=:\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "adds r8, r8, lr\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r7, r7, #0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r6, r6, #0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "adc r5, r5, #0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d5, r7, r8\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d4, r5, r6\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q11\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q0, q12\n\t" - "veor.32 q0, q0, q13\n\t" - "veor.32 q14, q14, q0\n\t" - "vst1.8 {q14}, [%[out]]!\n\t" - "mov lr, #1\n\t" - "vrev32.8 q0, q2\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_128_done_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_ctr_encrypt_arm32_crypto_128_partial_done_%=\n\t" - "ldr r4, [sp, #8]\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "adds r8, r8, lr\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r7, r7, #0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "adcs r6, r6, #0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "adc r5, r5, #0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d5, r7, r8\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "vmov d4, r5, r6\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q11\n\t" - "aesmc.8 q0, q0\n\t" - "ldr lr, [sp, #4]\n\t" - "aese.8 q0, q12\n\t" - "veor.32 q0, q0, q13\n\t" - "vst1.32 {q0}, [lr]\n\t" - "vmov q0, q2\n\t" - "mov r5, #16\n\t" - "sub r5, r5, %[sz]\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_128_start_byte_%=:\n\t" - "ldrb r7, [lr], #1\n\t" - "ldrb r8, [%[in]], #1\n\t" - "eor r7, r7, r8\n\t" - "subs %[sz], %[sz], #1\n\t" - "strb r7, [%[out]], #1\n\t" - "bgt L_aes_ctr_encrypt_arm32_crypto_128_start_byte_%=\n\t" - "vrev32.8 q0, q2\n\t" - "str r5, [r4]\n\t" - "\n" - "L_aes_ctr_encrypt_arm32_crypto_128_partial_done_%=:\n\t" -#endif /* !NO_AES_128 */ - "\n" - "L_aes_ctr_encrypt_arm32_crypto_done_%=:\n\t" - "vst1.32 {q0}, [%[reg]]\n\t" - "pop {%[key], %[tmp]}\n\t" - "pop {%[left], %[nr]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [reg] "+r" (reg), - [key] "+r" (key), [tmp] "+r" (tmp), [left] "+r" (left), - [nr] "+r" (nr) - : -#else - : - : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [reg] "r" (reg), - [key] "r" (key), [tmp] "r" (tmp), [left] "r" (left), [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r6", "r7", "r8", "q0", "q1", "q2", "q3", "q4", "q5", - "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" - ); -} - -#endif /* WOLFSSL_AES_COUNTER */ -#ifdef HAVE_AESGCM -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_GCM_set_key_AARCH32(const byte* nonce_p, - const byte* key_p, byte* gcm_h_p, int nr_p) -#else -WC_OMIT_FRAME_POINTER void AES_GCM_set_key_AARCH32(const byte* nonce, - const byte* key, byte* gcm_h, int nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* nonce __asm__ ("r0") = (const byte*)nonce_p; - register const byte* key __asm__ ("r1") = (const byte*)key_p; - register byte* gcm_h __asm__ ("r2") = (byte*)gcm_h_p; - register int nr __asm__ ("r3") = (int)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "vld1.8 {q0}, [%[nonce]]\n\t" - "vld1.8 {q1-q2}, [%[key]]!\n\t" - "vld1.8 {q3-q4}, [%[key]]!\n\t" - "aese.8 q0, q1\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q2\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.8 {q1-q2}, [%[key]]!\n\t" - "vld1.8 {q3-q4}, [%[key]]!\n\t" - "aese.8 q0, q1\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q2\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "subs %[nr], %[nr], #10\n\t" - "vld1.8 {q1-q2}, [%[key]]!\n\t" - "aese.8 q0, q1\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q2\n\t" - "beq L_aes_gcm_set_key_arm32_crypto_round_done_%=\n\t" - "vld1.8 {q1-q2}, [%[key]]!\n\t" - "subs %[nr], %[nr], #2\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q1\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q2\n\t" - "beq L_aes_gcm_set_key_arm32_crypto_round_done_%=\n\t" - "vld1.8 {q1-q2}, [%[key]]!\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q1\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q2\n\t" - "\n" - "L_aes_gcm_set_key_arm32_crypto_round_done_%=:\n\t" - "vld1.8 {q1}, [%[key]]\n\t" - "veor q0, q0, q1\n\t" - "vmov.i8 q1, #0x55\n\t" - "vshl.u8 q2, q0, #1\n\t" - "vshr.u8 q0, q0, #1\n\t" - "vbif.8 q0, q2, q1\n\t" - "vmov.i8 q1, #51\n\t" - "vshl.u8 q2, q0, #2\n\t" - "vshr.u8 q0, q0, #2\n\t" - "vbit.8 q2, q0, q1\n\t" - "vshl.u8 q0, q2, #4\n\t" - "vsri.u8 q0, q2, #4\n\t" - "vst1.32 {q0}, [%[gcm_h]]\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [nonce] "+r" (nonce), [key] "+r" (key), [gcm_h] "+r" (gcm_h), - [nr] "+r" (nr) - : -#else - : - : [nonce] "r" (nonce), [key] "r" (key), [gcm_h] "r" (gcm_h), - [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r12", "q0", "q1", "q2", "q3", "q4" - ); -} - -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_GCM_encrypt_AARCH32(const byte* in_p, - byte* out_p, word32 sz_p, const byte* nonce_p, word32 nonceSz_p, - byte* tag_p, word32 tagSz_p, const byte* aad_p, word32 aadSz_p, byte* key_p, - byte* gcm_h_p, byte* tmp_p, byte* reg_p, int nr_p) -#else -WC_OMIT_FRAME_POINTER void AES_GCM_encrypt_AARCH32(const byte* in, byte* out, - word32 sz, const byte* nonce, word32 nonceSz, byte* tag, word32 tagSz, - const byte* aad, word32 aadSz, byte* key, byte* gcm_h, byte* tmp, byte* reg, - int nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* in __asm__ ("r0") = (const byte*)in_p; - register byte* out __asm__ ("r1") = (byte*)out_p; - register word32 sz __asm__ ("r2") = (word32)sz_p; - register const byte* nonce __asm__ ("r3") = (const byte*)nonce_p; - register word32 nonceSz __asm__ ("r12") = (word32)nonceSz_p; - register byte* tag __asm__ ("lr") = (byte*)tag_p; - register word32 tagSz __asm__ ("r4") = (word32)tagSz_p; - register const byte* aad __asm__ ("r5") = (const byte*)aad_p; - register word32 aadSz __asm__ ("r6") = (word32)aadSz_p; - register byte* key __asm__ ("r7") = (byte*)key_p; - register byte* gcm_h __asm__ ("r8") = (byte*)gcm_h_p; - register byte* tmp __asm__ ("r9") = (byte*)tmp_p; - register byte* reg __asm__ ("r10") = (byte*)reg_p; - register int nr __asm__ ("r11") = (int)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[tagSz], %[aad], %[aadSz], %[key], %[gcm_h], %[tmp], %[reg], %[nr]}\n\t" - "push {%[nonceSz], %[tag]}\n\t" - /* key */ - "ldr r7, [sp, #20]\n\t" - /* tmp */ - "ldr r9, [sp, #28]\n\t" - /* nonceSz */ - "ldr r12, [sp]\n\t" - "cmp r12, #12\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_nonce_setup_done_%=\n\t" - "vmov.i8 q13, #0x87\n\t" - /* gcm_h */ - "ldr r8, [sp, #24]\n\t" - "veor.8 q6, q6, q6\n\t" - "vshr.u64 q13, q13, #56\n\t" - "vld1.32 {q8}, [r8]\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_nonce_setup_done_%=:\n\t" - /* Load Nonce */ - "cmp r12, #12\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_ghash_nonce_%=\n\t" - "ldr r5, [%[nonce]]\n\t" - "ldr r8, [%[nonce], #4]\n\t" - "ldr r12, [%[nonce], #8]\n\t" - "vmov.i32 q6, #0x1000000\n\t" - "vmov.32 s24, r5\n\t" - "vmov.32 s25, r8\n\t" - "vmov.32 s26, r12\n\t" - "mov r5, #1\n\t" - "b L_aes_gcm_encrypt_arm32_crypto_done_nonce_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_ghash_nonce_%=:\n\t" - "lsr r10, r12, #4\n\t" - "cmp r10, #0\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_nonce_done_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_nonce_start_1_%=:\n\t" - "vld1.32 {q14}, [%[nonce]]!\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q6, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q6, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q6\n\t" - /* Reduce */ - "vmull.p64 q6, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d13\n\t" - "veor.8 d1, d1, d12\n\t" - "vmull.p64 q6, d2, d26\n\t" - "veor.8 q6, q6, q0\n\t" - /* Done GHASH */ - "subs r10, r10, #1\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_nonce_start_1_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_nonce_done_%=:\n\t" - "ands r11, r12, #15\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_nonce_partial_done_%=\n\t" - "veor.8 q0, q0, q0\n\t" - "mov r12, r11\n\t" - "vst1.32 {q0}, [r9]\n\t" - "cmp r12, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_nonce_start_sw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_nonce_start_dw_%=:\n\t" - "ldr r8, [%[nonce]], #4\n\t" - "sub r12, r12, #4\n\t" - "str r8, [r9], #4\n\t" - "cmp r12, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_nonce_start_dw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_nonce_start_sw_%=:\n\t" - "cmp r12, #2\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte_%=\n\t" - "ldrh r8, [%[nonce]], #2\n\t" - "sub r12, r12, #2\n\t" - "strh r8, [r9], #2\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte_%=:\n\t" - "cmp r12, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_nonce_end_bytes_%=\n\t" - "ldrb r8, [%[nonce]], #1\n\t" - "subs r12, r12, #1\n\t" - "strb r8, [r9], #1\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_nonce_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "vld1.32 {q14}, [r9]\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q6, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q6, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q6\n\t" - /* Reduce */ - "vmull.p64 q6, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d13\n\t" - "veor.8 d1, d1, d12\n\t" - "vmull.p64 q6, d2, d26\n\t" - "veor.8 q6, q6, q0\n\t" - /* Done GHASH */ - "\n" - "L_aes_gcm_encrypt_arm32_crypto_nonce_partial_done_%=:\n\t" - "veor.8 q0, q0, q0\n\t" - /* nonceSz */ - "ldr r12, [sp]\n\t" - "lsr r10, r12, #29\n\t" - "lsl r11, r12, #3\n\t" - "rbit r10, r10\n\t" - "rbit r11, r11\n\t" - "vmov.32 s2, r10\n\t" - "vmov.32 s3, r11\n\t" - "veor.8 q6, q6, q0\n\t" - "vmull.p64 q2, d13, d16\n\t" - "vmull.p64 q12, d12, d17\n\t" - "vmull.p64 q0, d12, d16\n\t" - "vmull.p64 q1, d13, d17\n\t" - "veor.8 q2, q2, q12\n\t" - "vmull.p64 q6, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d13\n\t" - "veor.8 d1, d1, d12\n\t" - "vmull.p64 q6, d2, d26\n\t" - "veor.8 q6, q6, q0\n\t" - "vmov.32 r5, s24\n\t" - "vmov.32 r8, s25\n\t" - "vmov.32 r12, s26\n\t" - "rbit r5, r5\n\t" - "rbit r8, r8\n\t" - "rbit r12, r12\n\t" - "rev r5, r5\n\t" - "rev r8, r8\n\t" - "rev r12, r12\n\t" - "vmov.32 s24, r5\n\t" - "vmov.32 s25, r8\n\t" - "vmov.32 s26, r12\n\t" - "vmov.32 r5, s27\n\t" - "rbit r5, r5\n\t" - "rev r5, r5\n\t" - "vmov.32 s27, r5\n\t" - "rev r5, r5\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_done_nonce_%=:\n\t" - "vldm.32 r7!, {q0-q3}\n\t" - "vldm.32 r7!, {q7-q13}\n\t" - /* nr */ - "ldr r12, [sp, #36]\n\t" - "lsr r10, %[sz], #4\n\t" - "cmp r12, #12\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_start_128_%=\n\t" - "bgt L_aes_gcm_encrypt_arm32_crypto_start_256_%=\n\t" - /* AES_GCM_192 */ -#ifndef NO_AES_192 - "cmp r10, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_192_done_%=\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_192_start_1_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_start_2_%=:\n\t" - "add r8, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "add r5, r5, #2\n\t" - "vmov.8 q5, q6\n\t" - "rev r8, r8\n\t" - "rev r12, r5\n\t" - "vmov s19, r8\n\t" - "vmov s23, r12\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q0\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q1\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q2\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q3\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q7\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q8\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q9\n\t" - "aesmc.8 q5, q5\n\t" - "subs r10, r10, #2\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q10\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q11\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q12\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q13\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "aese.8 q5, q14\n\t" - "veor.8 q5, q5, q15\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "sub r7, r7, #16\n\t" - "veor.8 q14, q14, q4\n\t" - "veor.8 q15, q15, q5\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "cmp r10, #1\n\t" - "bgt L_aes_gcm_encrypt_arm32_crypto_192_start_2_%=\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_192_done_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_start_1_%=:\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "sub r7, r7, #16\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [%[out]]!\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_done_%=:\n\t" - "ands r11, %[sz], #15\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_192_partial_done_%=\n\t" - "veor.8 q14, q14, q14\n\t" - "mov r4, r11\n\t" - "vst1.32 {q14}, [r9]\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_192_start_sw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_start_dw_%=:\n\t" - "ldr lr, [%[in]], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [r9], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_192_start_dw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_192_start_byte_%=\n\t" - "ldrh lr, [%[in]], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [r9], #2\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_192_end_bytes_%=\n\t" - "ldrb lr, [%[in]], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [r9], #1\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_192_start_byte_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "vld1.8 {q14}, [r9]\n\t" - "sub r7, r7, #16\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [r9]\n\t" - "mov r4, r11\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_192_out_start_sw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_out_start_dw_%=:\n\t" - "ldr lr, [r9], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [%[out]], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_192_out_start_dw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_out_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte_%=\n\t" - "ldrh lr, [r9], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [%[out]], #2\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_192_out_end_bytes_%=\n\t" - "ldrb lr, [r9], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [%[out]], #1\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_out_end_bytes_%=:\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_192_partial_done_%=:\n\t" - /* Finish */ - "add r8, %[sz], #15\n\t" - "sub r8, r5, r8, lsr #4\n\t" - "rev r8, r8\n\t" - "vmov.32 s27, r8\n\t" - "aese.8 q6, q0\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q1\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q2\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q3\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q7\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q8\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q9\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q10\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q11\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q12\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q6, q13\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q6, q14\n\t" - "veor.8 q6, q6, q15\n\t" - "sub r7, r7, #16\n\t" -#endif /* !NO_AES_192 */ - "b L_aes_gcm_encrypt_arm32_crypto_done_enc_%=\n\t" - /* AES_GCM_256 */ - "\n" - "L_aes_gcm_encrypt_arm32_crypto_start_256_%=:\n\t" -#ifndef NO_AES_256 - "cmp r10, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_256_done_%=\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_256_start_1_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_start_2_%=:\n\t" - "add r8, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "add r5, r5, #2\n\t" - "vmov.8 q5, q6\n\t" - "rev r8, r8\n\t" - "rev r12, r5\n\t" - "vmov s19, r8\n\t" - "vmov s23, r12\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q0\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q1\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q2\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q3\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q7\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q8\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q9\n\t" - "aesmc.8 q5, q5\n\t" - "subs r10, r10, #2\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q10\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q11\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q12\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q13\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q15}, [r7]!\n\t" - "aese.8 q4, q14\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q14\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q15\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q15\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "aese.8 q5, q14\n\t" - "veor.8 q5, q5, q15\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "sub r7, r7, #48\n\t" - "veor.8 q14, q14, q4\n\t" - "veor.8 q15, q15, q5\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "cmp r10, #1\n\t" - "bgt L_aes_gcm_encrypt_arm32_crypto_256_start_2_%=\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_256_done_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_start_1_%=:\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]!\n\t" - "aese.8 q4, q14\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q15\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "sub r7, r7, #48\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [%[out]]!\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_done_%=:\n\t" - "ands r11, %[sz], #15\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_256_partial_done_%=\n\t" - "veor.8 q14, q14, q14\n\t" - "mov r4, r11\n\t" - "vst1.32 {q14}, [r9]\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_256_start_sw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_start_dw_%=:\n\t" - "ldr lr, [%[in]], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [r9], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_256_start_dw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_256_start_byte_%=\n\t" - "ldrh lr, [%[in]], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [r9], #2\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_256_end_bytes_%=\n\t" - "ldrb lr, [%[in]], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [r9], #1\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_256_start_byte_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]!\n\t" - "aese.8 q4, q14\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q15\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "vld1.8 {q14}, [r9]\n\t" - "sub r7, r7, #48\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [r9]\n\t" - "mov r4, r11\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_256_out_start_sw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_out_start_dw_%=:\n\t" - "ldr lr, [r9], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [%[out]], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_256_out_start_dw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_out_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte_%=\n\t" - "ldrh lr, [r9], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [%[out]], #2\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_256_out_end_bytes_%=\n\t" - "ldrb lr, [r9], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [%[out]], #1\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_out_end_bytes_%=:\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_256_partial_done_%=:\n\t" - /* Finish */ - "add r8, %[sz], #15\n\t" - "sub r8, r5, r8, lsr #4\n\t" - "rev r8, r8\n\t" - "vmov.32 s27, r8\n\t" - "aese.8 q6, q0\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q1\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q2\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q3\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q7\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q8\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q9\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q10\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q11\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q12\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q6, q13\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q15}, [r7]!\n\t" - "aese.8 q6, q14\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q6, q15\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q6, q14\n\t" - "veor.8 q6, q6, q15\n\t" - "sub r7, r7, #48\n\t" -#endif /* !NO_AES_256 */ - "b L_aes_gcm_encrypt_arm32_crypto_done_enc_%=\n\t" - /* AES_GCM_128 */ - "\n" - "L_aes_gcm_encrypt_arm32_crypto_start_128_%=:\n\t" -#ifndef NO_AES_128 - "cmp r10, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_128_done_%=\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_128_start_1_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_start_2_%=:\n\t" - "add r8, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "add r5, r5, #2\n\t" - "vmov.8 q5, q6\n\t" - "rev r8, r8\n\t" - "rev r12, r5\n\t" - "vmov s19, r8\n\t" - "vmov s23, r12\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q0\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q1\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q2\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q3\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q7\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q8\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q9\n\t" - "aesmc.8 q5, q5\n\t" - "subs r10, r10, #2\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q10\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q11\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "aese.8 q4, q12\n\t" - "veor.8 q4, q4, q13\n\t" - "aese.8 q5, q12\n\t" - "veor.8 q5, q5, q13\n\t" - "veor.8 q14, q14, q4\n\t" - "veor.8 q15, q15, q5\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "cmp r10, #1\n\t" - "bgt L_aes_gcm_encrypt_arm32_crypto_128_start_2_%=\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_128_done_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_start_1_%=:\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q4, q12\n\t" - "veor.8 q4, q4, q13\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [%[out]]!\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_done_%=:\n\t" - "ands r11, %[sz], #15\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_128_partial_done_%=\n\t" - "veor.8 q14, q14, q14\n\t" - "mov r4, r11\n\t" - "vst1.32 {q14}, [r9]\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_128_start_sw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_start_dw_%=:\n\t" - "ldr lr, [%[in]], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [r9], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_128_start_dw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_128_start_byte_%=\n\t" - "ldrh lr, [%[in]], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [r9], #2\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_128_end_bytes_%=\n\t" - "ldrb lr, [%[in]], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [r9], #1\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_128_start_byte_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.8 {q14}, [r9]\n\t" - "aese.8 q4, q12\n\t" - "veor.8 q4, q4, q13\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [r9]\n\t" - "mov r4, r11\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_128_out_start_sw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_out_start_dw_%=:\n\t" - "ldr lr, [r9], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [%[out]], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_128_out_start_dw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_out_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte_%=\n\t" - "ldrh lr, [r9], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [%[out]], #2\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_128_out_end_bytes_%=\n\t" - "ldrb lr, [r9], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [%[out]], #1\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_out_end_bytes_%=:\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_128_partial_done_%=:\n\t" - /* Finish */ - "add r8, %[sz], #15\n\t" - "sub r8, r5, r8, lsr #4\n\t" - "rev r8, r8\n\t" - "vmov.32 s27, r8\n\t" - "aese.8 q6, q0\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q1\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q2\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q3\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q7\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q8\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q9\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q10\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q11\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q12\n\t" - "veor.8 q6, q6, q13\n\t" -#endif /* !NO_AES_128 */ - "\n" - "L_aes_gcm_encrypt_arm32_crypto_done_enc_%=:\n\t" - /* aadSz */ - "ldr r6, [sp, #16]\n\t" - /* gcm_h */ - "ldr r8, [sp, #24]\n\t" - "sub %[out], %[out], %[sz]\n\t" - "vmov.i8 q13, #0x87\n\t" - "veor.8 q7, q7, q7\n\t" - "vshr.u64 q13, q13, #56\n\t" - "vld1.32 {q8}, [r8]\n\t" - "orr r10, r6, %[sz]\n\t" - "cmp r10, #32\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_h_done_%=\n\t" - /* Square H => H^2 */ - "vmull.p64 q1, d17, d17\n\t" - "vmull.p64 q0, d16, d16\n\t" - "vmull.p64 q9, d3, d27\n\t" - "veor.8 d2, d2, d19\n\t" - "veor.8 d1, d1, d18\n\t" - "vmull.p64 q9, d2, d26\n\t" - "veor.8 q9, q9, q0\n\t" - "cmp r10, #0x40\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_h_done_%=\n\t" - /* Multiply H and H^2 => H^3 */ - "vmull.p64 q2, d17, d18\n\t" - "vmull.p64 q3, d16, d19\n\t" - "vmull.p64 q0, d16, d18\n\t" - "vmull.p64 q1, d17, d19\n\t" - "veor.8 q2, q2, q3\n\t" - /* Reduce */ - "vmull.p64 q10, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d21\n\t" - "veor.8 d1, d1, d20\n\t" - "vmull.p64 q10, d2, d26\n\t" - "veor.8 q10, q10, q0\n\t" - /* Square H^2 => H^4 */ - "vmull.p64 q1, d19, d19\n\t" - "vmull.p64 q0, d18, d18\n\t" - "vmull.p64 q11, d3, d27\n\t" - "veor.8 d2, d2, d23\n\t" - "veor.8 d1, d1, d22\n\t" - "vmull.p64 q11, d2, d26\n\t" - "veor.8 q11, q11, q0\n\t" - /* Done */ - "\n" - "L_aes_gcm_encrypt_arm32_crypto_h_done_%=:\n\t" - /* aad */ - "ldr r5, [sp, #12]\n\t" - "lsr r10, r6, #4\n\t" - "cmp r10, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_aad_done_%=\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_aad_start_1_%=\n\t" - "cmp r10, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_aad_start_2_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_aad_start_4_%=:\n\t" - "vldm r5!, {q0-q2}\n\t" - "vmov.i8 q12, #0x55\n\t" - "vmov.i8 q5, #51\n\t" - "vshl.u8 q14, q0, #1\n\t" - "vshl.u8 q15, q1, #1\n\t" - "vshl.u8 q4, q2, #1\n\t" - "ldr lr, [r5], #4\n\t" - "vshr.u8 q0, q0, #1\n\t" - "vshr.u8 q1, q1, #1\n\t" - "vshr.u8 q2, q2, #1\n\t" - "ldr r4, [r5], #4\n\t" - "vbit.8 q14, q0, q12\n\t" - "vbit.8 q15, q1, q12\n\t" - "vbit.8 q4, q2, q12\n\t" - "ldr r8, [r5], #4\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshl.u8 q1, q15, #2\n\t" - "vshl.u8 q2, q4, #2\n\t" - "ldr r12, [r5], #4\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vshr.u8 q15, q15, #2\n\t" - "vshr.u8 q4, q4, #2\n\t" - "rbit lr, lr\n\t" - "vbit.8 q0, q14, q5\n\t" - "rbit r4, r4\n\t" - "vbit.8 q1, q15, q5\n\t" - "rbit r8, r8\n\t" - "vbit.8 q2, q4, q5\n\t" - "rbit r12, r12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "rev lr, lr\n\t" - "vshl.u8 q15, q1, #4\n\t" - "rev r4, r4\n\t" - "vshl.u8 q4, q2, #4\n\t" - "rev r8, r8\n\t" - "vsri.u8 q14, q0, #4\n\t" - "rev r12, r12\n\t" - "vsri.u8 q15, q1, #4\n\t" - "vsri.u8 q4, q2, #4\n\t" - "vmov d10, lr, r4\n\t" - "vmov d11, r8, r12\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d11, d16\n\t" - "vmull.p64 q7, d10, d17\n\t" - "vmull.p64 q0, d10, d16\n\t" - "vmull.p64 q1, d11, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^2 */ - "vmull.p64 q3, d8, d18\n\t" - "vmull.p64 q7, d9, d19\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d9, d18\n\t" - "vmull.p64 q7, d8, d19\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^3 */ - "vmull.p64 q3, d30, d20\n\t" - "vmull.p64 q7, d31, d21\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d31, d20\n\t" - "vmull.p64 q7, d30, d21\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^4 */ - "vmull.p64 q3, d24, d22\n\t" - "vmull.p64 q7, d25, d23\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d25, d22\n\t" - "vmull.p64 q7, d24, d23\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "sub r10, r10, #4\n\t" - "cmp r10, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_aad_start_4_%=\n\t" - "cmp r10, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_aad_done_%=\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_aad_start_1_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_aad_start_2_%=:\n\t" - "vld1.32 {q14-q15}, [r5]!\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshl.u8 q1, q15, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vshr.u8 q15, q15, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vbif.8 q15, q1, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshl.u8 q1, q15, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vshr.u8 q15, q15, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vbit.8 q1, q15, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vshl.u8 q15, q1, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "vsri.u8 q15, q1, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d31, d16\n\t" - "vmull.p64 q7, d30, d17\n\t" - "vmull.p64 q0, d30, d16\n\t" - "vmull.p64 q1, d31, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^2 */ - "vmull.p64 q3, d24, d18\n\t" - "vmull.p64 q7, d25, d19\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d25, d18\n\t" - "vmull.p64 q7, d24, d19\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "sub r10, r10, #2\n\t" - "cmp r10, #0\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_aad_done_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_aad_start_1_%=:\n\t" - "vld1.32 {q14}, [r5]!\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q7, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "\n" - "L_aes_gcm_encrypt_arm32_crypto_aad_done_%=:\n\t" - "ands r11, r6, #15\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_aad_partial_done_%=\n\t" - "veor.8 q0, q0, q0\n\t" - "mov r12, r11\n\t" - "vst1.32 {q0}, [r9]\n\t" - "cmp r12, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_aad_start_sw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_aad_start_dw_%=:\n\t" - "ldr r8, [r5], #4\n\t" - "sub r12, r12, #4\n\t" - "str r8, [r9], #4\n\t" - "cmp r12, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_aad_start_dw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_aad_start_sw_%=:\n\t" - "cmp r12, #2\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_aad_start_byte_%=\n\t" - "ldrh r8, [r5], #2\n\t" - "sub r12, r12, #2\n\t" - "strh r8, [r9], #2\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_aad_start_byte_%=:\n\t" - "cmp r12, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_aad_end_bytes_%=\n\t" - "ldrb r8, [r5], #1\n\t" - "subs r12, r12, #1\n\t" - "strb r8, [r9], #1\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_aad_start_byte_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_aad_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "vld1.32 {q14}, [r9]\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q7, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "\n" - "L_aes_gcm_encrypt_arm32_crypto_aad_partial_done_%=:\n\t" - /* out */ - "lsr r10, %[sz], #4\n\t" - "cmp r10, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_out_done_%=\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_out_start_1_%=\n\t" - "cmp r10, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_out_start_2_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_out_start_4_%=:\n\t" - "vldm %[out]!, {q0-q2}\n\t" - "vmov.i8 q12, #0x55\n\t" - "vmov.i8 q5, #51\n\t" - "vshl.u8 q14, q0, #1\n\t" - "vshl.u8 q15, q1, #1\n\t" - "vshl.u8 q4, q2, #1\n\t" - "ldr lr, [%[out]], #4\n\t" - "vshr.u8 q0, q0, #1\n\t" - "vshr.u8 q1, q1, #1\n\t" - "vshr.u8 q2, q2, #1\n\t" - "ldr r4, [%[out]], #4\n\t" - "vbit.8 q14, q0, q12\n\t" - "vbit.8 q15, q1, q12\n\t" - "vbit.8 q4, q2, q12\n\t" - "ldr r8, [%[out]], #4\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshl.u8 q1, q15, #2\n\t" - "vshl.u8 q2, q4, #2\n\t" - "ldr r12, [%[out]], #4\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vshr.u8 q15, q15, #2\n\t" - "vshr.u8 q4, q4, #2\n\t" - "rbit lr, lr\n\t" - "vbit.8 q0, q14, q5\n\t" - "rbit r4, r4\n\t" - "vbit.8 q1, q15, q5\n\t" - "rbit r8, r8\n\t" - "vbit.8 q2, q4, q5\n\t" - "rbit r12, r12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "rev lr, lr\n\t" - "vshl.u8 q15, q1, #4\n\t" - "rev r4, r4\n\t" - "vshl.u8 q4, q2, #4\n\t" - "rev r8, r8\n\t" - "vsri.u8 q14, q0, #4\n\t" - "rev r12, r12\n\t" - "vsri.u8 q15, q1, #4\n\t" - "vsri.u8 q4, q2, #4\n\t" - "vmov d10, lr, r4\n\t" - "vmov d11, r8, r12\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d11, d16\n\t" - "vmull.p64 q7, d10, d17\n\t" - "vmull.p64 q0, d10, d16\n\t" - "vmull.p64 q1, d11, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^2 */ - "vmull.p64 q3, d8, d18\n\t" - "vmull.p64 q7, d9, d19\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d9, d18\n\t" - "vmull.p64 q7, d8, d19\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^3 */ - "vmull.p64 q3, d30, d20\n\t" - "vmull.p64 q7, d31, d21\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d31, d20\n\t" - "vmull.p64 q7, d30, d21\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^4 */ - "vmull.p64 q3, d24, d22\n\t" - "vmull.p64 q7, d25, d23\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d25, d22\n\t" - "vmull.p64 q7, d24, d23\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "sub r10, r10, #4\n\t" - "cmp r10, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_out_start_4_%=\n\t" - "cmp r10, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_out_done_%=\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_out_start_1_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_out_start_2_%=:\n\t" - "vld1.32 {q14-q15}, [%[out]]!\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshl.u8 q1, q15, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vshr.u8 q15, q15, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vbif.8 q15, q1, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshl.u8 q1, q15, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vshr.u8 q15, q15, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vbit.8 q1, q15, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vshl.u8 q15, q1, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "vsri.u8 q15, q1, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d31, d16\n\t" - "vmull.p64 q7, d30, d17\n\t" - "vmull.p64 q0, d30, d16\n\t" - "vmull.p64 q1, d31, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^2 */ - "vmull.p64 q3, d24, d18\n\t" - "vmull.p64 q7, d25, d19\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d25, d18\n\t" - "vmull.p64 q7, d24, d19\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "sub r10, r10, #2\n\t" - "cmp r10, #0\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_out_done_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_out_start_1_%=:\n\t" - "vld1.32 {q14}, [%[out]]!\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q7, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "\n" - "L_aes_gcm_encrypt_arm32_crypto_out_done_%=:\n\t" - "ands r11, %[sz], #15\n\t" - "beq L_aes_gcm_encrypt_arm32_crypto_out_partial_done_%=\n\t" - "veor.8 q0, q0, q0\n\t" - "mov r12, r11\n\t" - "vst1.32 {q0}, [r9]\n\t" - "cmp r12, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_out_start_sw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_out_start_dw_%=:\n\t" - "ldr r8, [%[out]], #4\n\t" - "sub r12, r12, #4\n\t" - "str r8, [r9], #4\n\t" - "cmp r12, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_out_start_dw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_out_start_sw_%=:\n\t" - "cmp r12, #2\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_out_start_byte_%=\n\t" - "ldrh r8, [%[out]], #2\n\t" - "sub r12, r12, #2\n\t" - "strh r8, [r9], #2\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_out_start_byte_%=:\n\t" - "cmp r12, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_out_end_bytes_%=\n\t" - "ldrb r8, [%[out]], #1\n\t" - "subs r12, r12, #1\n\t" - "strb r8, [r9], #1\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_out_start_byte_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_out_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "vld1.32 {q14}, [r9]\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q7, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "\n" - "L_aes_gcm_encrypt_arm32_crypto_out_partial_done_%=:\n\t" - "lsr lr, r6, #29\n\t" - "lsl r6, r6, #3\n\t" - "rbit lr, lr\n\t" - "rbit r6, r6\n\t" - "vmov s0, lr\n\t" - "vmov s1, r6\n\t" - "lsr lr, %[sz], #29\n\t" - "lsl %[sz], %[sz], #3\n\t" - "rbit lr, lr\n\t" - "rbit %[sz], %[sz]\n\t" - "vmov s2, lr\n\t" - "vmov s3, %[sz]\n\t" - "veor.8 q7, q7, q0\n\t" - "vmull.p64 q2, d15, d16\n\t" - "vmull.p64 q3, d14, d17\n\t" - "vmull.p64 q0, d14, d16\n\t" - "vmull.p64 q1, d15, d17\n\t" - "veor.8 q2, q2, q3\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - "vmov.i8 q0, #0x55\n\t" - "vshl.u8 q1, q7, #1\n\t" - "vshr.u8 q7, q7, #1\n\t" - "vbif.8 q7, q1, q0\n\t" - "vmov.i8 q0, #51\n\t" - "vshl.u8 q1, q7, #2\n\t" - "vshr.u8 q7, q7, #2\n\t" - "vbit.8 q1, q7, q0\n\t" - "vshl.u8 q7, q1, #4\n\t" - "vsri.u8 q7, q1, #4\n\t" - /* DONE */ - "veor.8 q7, q7, q6\n\t" - /* tag */ - "ldr lr, [sp, #4]\n\t" - /* tagSz */ - "ldr r4, [sp, #8]\n\t" - "cmp r4, #16\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_tag_tag_partial_%=\n\t" - "vst1.8 {q7}, [lr]\n\t" - "b L_aes_gcm_encrypt_arm32_crypto_done_gcm_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_tag_tag_partial_%=:\n\t" - "vst1.8 {q7}, [r9]\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_sw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_dw_%=:\n\t" - "ldr r8, [r9], #4\n\t" - "sub r4, r4, #4\n\t" - "str r8, [lr], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_dw_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte_%=\n\t" - "ldrh r8, [r9], #2\n\t" - "sub r4, r4, #2\n\t" - "strh r8, [lr], #2\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_end_bytes_%=\n\t" - "ldrb r8, [r9], #1\n\t" - "subs r4, r4, #1\n\t" - "strb r8, [lr], #1\n\t" - "bne L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte_%=\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_tag_tag_end_bytes_%=:\n\t" - "\n" - "L_aes_gcm_encrypt_arm32_crypto_done_gcm_%=:\n\t" - "pop {%[nonceSz], %[tag]}\n\t" - "pop {%[tagSz], %[aad], %[aadSz], %[key], %[gcm_h], %[tmp], %[reg], %[nr]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), - [nonce] "+r" (nonce), [nonceSz] "+r" (nonceSz), [tag] "+r" (tag), - [tagSz] "+r" (tagSz), [aad] "+r" (aad), [aadSz] "+r" (aadSz), - [key] "+r" (key), [gcm_h] "+r" (gcm_h), [tmp] "+r" (tmp), - [reg] "+r" (reg), [nr] "+r" (nr) - : -#else - : - : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [nonce] "r" (nonce), - [nonceSz] "r" (nonceSz), [tag] "r" (tag), [tagSz] "r" (tagSz), - [aad] "r" (aad), [aadSz] "r" (aadSz), [key] "r" (key), - [gcm_h] "r" (gcm_h), [tmp] "r" (tmp), [reg] "r" (reg), [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", - "q9", "q10", "q11", "q12", "q13", "q14", "q15" - ); -} - -#ifdef HAVE_AES_DECRYPT -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER int AES_GCM_decrypt_AARCH32(const byte* in_p, byte* out_p, - word32 sz_p, const byte* nonce_p, word32 nonceSz_p, const byte* tag_p, - word32 tagSz_p, const byte* aad_p, word32 aadSz_p, byte* key_p, - byte* gcm_h_p, byte* tmp_p, byte* reg_p, int nr_p) -#else -WC_OMIT_FRAME_POINTER int AES_GCM_decrypt_AARCH32(const byte* in, byte* out, - word32 sz, const byte* nonce, word32 nonceSz, const byte* tag, word32 tagSz, - const byte* aad, word32 aadSz, byte* key, byte* gcm_h, byte* tmp, byte* reg, - int nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* in __asm__ ("r0") = (const byte*)in_p; - register byte* out __asm__ ("r1") = (byte*)out_p; - register word32 sz __asm__ ("r2") = (word32)sz_p; - register const byte* nonce __asm__ ("r3") = (const byte*)nonce_p; - register word32 nonceSz __asm__ ("r12") = (word32)nonceSz_p; - register const byte* tag __asm__ ("lr") = (const byte*)tag_p; - register word32 tagSz __asm__ ("r4") = (word32)tagSz_p; - register const byte* aad __asm__ ("r5") = (const byte*)aad_p; - register word32 aadSz __asm__ ("r6") = (word32)aadSz_p; - register byte* key __asm__ ("r7") = (byte*)key_p; - register byte* gcm_h __asm__ ("r8") = (byte*)gcm_h_p; - register byte* tmp __asm__ ("r9") = (byte*)tmp_p; - register byte* reg __asm__ ("r10") = (byte*)reg_p; - register int nr __asm__ ("r11") = (int)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[tagSz], %[aad], %[aadSz], %[key], %[gcm_h], %[tmp], %[reg], %[nr]}\n\t" - "push {%[nonceSz], %[tag]}\n\t" - /* key */ - "ldr r7, [sp, #20]\n\t" - /* tmp */ - "ldr r9, [sp, #28]\n\t" - /* aadSz */ - "ldr r6, [sp, #16]\n\t" - /* gcm_h */ - "ldr r8, [sp, #24]\n\t" - "vmov.i8 q13, #0x87\n\t" - "veor.8 q7, q7, q7\n\t" - "vshr.u64 q13, q13, #56\n\t" - "vld1.32 {q8}, [r8]\n\t" - "orr r10, r6, %[sz]\n\t" - "cmp r10, #32\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_h_done_%=\n\t" - /* Square H => H^2 */ - "vmull.p64 q1, d17, d17\n\t" - "vmull.p64 q0, d16, d16\n\t" - "vmull.p64 q9, d3, d27\n\t" - "veor.8 d2, d2, d19\n\t" - "veor.8 d1, d1, d18\n\t" - "vmull.p64 q9, d2, d26\n\t" - "veor.8 q9, q9, q0\n\t" - "cmp r10, #0x40\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_h_done_%=\n\t" - /* Multiply H and H^2 => H^3 */ - "vmull.p64 q2, d17, d18\n\t" - "vmull.p64 q3, d16, d19\n\t" - "vmull.p64 q0, d16, d18\n\t" - "vmull.p64 q1, d17, d19\n\t" - "veor.8 q2, q2, q3\n\t" - /* Reduce */ - "vmull.p64 q10, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d21\n\t" - "veor.8 d1, d1, d20\n\t" - "vmull.p64 q10, d2, d26\n\t" - "veor.8 q10, q10, q0\n\t" - /* Square H^2 => H^4 */ - "vmull.p64 q1, d19, d19\n\t" - "vmull.p64 q0, d18, d18\n\t" - "vmull.p64 q11, d3, d27\n\t" - "veor.8 d2, d2, d23\n\t" - "veor.8 d1, d1, d22\n\t" - "vmull.p64 q11, d2, d26\n\t" - "veor.8 q11, q11, q0\n\t" - /* Done */ - "\n" - "L_aes_gcm_decrypt_arm32_crypto_h_done_%=:\n\t" - /* aad */ - "ldr r5, [sp, #12]\n\t" - "lsr r10, r6, #4\n\t" - "cmp r10, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_aad_done_%=\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_aad_start_1_%=\n\t" - "cmp r10, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_aad_start_2_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_aad_start_4_%=:\n\t" - "vldm r5!, {q0-q2}\n\t" - "vmov.i8 q12, #0x55\n\t" - "vmov.i8 q5, #51\n\t" - "vshl.u8 q14, q0, #1\n\t" - "vshl.u8 q15, q1, #1\n\t" - "vshl.u8 q4, q2, #1\n\t" - "ldr lr, [r5], #4\n\t" - "vshr.u8 q0, q0, #1\n\t" - "vshr.u8 q1, q1, #1\n\t" - "vshr.u8 q2, q2, #1\n\t" - "ldr r4, [r5], #4\n\t" - "vbit.8 q14, q0, q12\n\t" - "vbit.8 q15, q1, q12\n\t" - "vbit.8 q4, q2, q12\n\t" - "ldr r8, [r5], #4\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshl.u8 q1, q15, #2\n\t" - "vshl.u8 q2, q4, #2\n\t" - "ldr r12, [r5], #4\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vshr.u8 q15, q15, #2\n\t" - "vshr.u8 q4, q4, #2\n\t" - "rbit lr, lr\n\t" - "vbit.8 q0, q14, q5\n\t" - "rbit r4, r4\n\t" - "vbit.8 q1, q15, q5\n\t" - "rbit r8, r8\n\t" - "vbit.8 q2, q4, q5\n\t" - "rbit r12, r12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "rev lr, lr\n\t" - "vshl.u8 q15, q1, #4\n\t" - "rev r4, r4\n\t" - "vshl.u8 q4, q2, #4\n\t" - "rev r8, r8\n\t" - "vsri.u8 q14, q0, #4\n\t" - "rev r12, r12\n\t" - "vsri.u8 q15, q1, #4\n\t" - "vsri.u8 q4, q2, #4\n\t" - "vmov d10, lr, r4\n\t" - "vmov d11, r8, r12\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d11, d16\n\t" - "vmull.p64 q7, d10, d17\n\t" - "vmull.p64 q0, d10, d16\n\t" - "vmull.p64 q1, d11, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^2 */ - "vmull.p64 q3, d8, d18\n\t" - "vmull.p64 q7, d9, d19\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d9, d18\n\t" - "vmull.p64 q7, d8, d19\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^3 */ - "vmull.p64 q3, d30, d20\n\t" - "vmull.p64 q7, d31, d21\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d31, d20\n\t" - "vmull.p64 q7, d30, d21\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^4 */ - "vmull.p64 q3, d24, d22\n\t" - "vmull.p64 q7, d25, d23\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d25, d22\n\t" - "vmull.p64 q7, d24, d23\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "sub r10, r10, #4\n\t" - "cmp r10, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_aad_start_4_%=\n\t" - "cmp r10, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_aad_done_%=\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_aad_start_1_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_aad_start_2_%=:\n\t" - "vld1.32 {q14-q15}, [r5]!\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshl.u8 q1, q15, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vshr.u8 q15, q15, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vbif.8 q15, q1, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshl.u8 q1, q15, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vshr.u8 q15, q15, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vbit.8 q1, q15, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vshl.u8 q15, q1, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "vsri.u8 q15, q1, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d31, d16\n\t" - "vmull.p64 q7, d30, d17\n\t" - "vmull.p64 q0, d30, d16\n\t" - "vmull.p64 q1, d31, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^2 */ - "vmull.p64 q3, d24, d18\n\t" - "vmull.p64 q7, d25, d19\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d25, d18\n\t" - "vmull.p64 q7, d24, d19\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "sub r10, r10, #2\n\t" - "cmp r10, #0\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_aad_done_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_aad_start_1_%=:\n\t" - "vld1.32 {q14}, [r5]!\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q7, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "\n" - "L_aes_gcm_decrypt_arm32_crypto_aad_done_%=:\n\t" - "ands r11, r6, #15\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_aad_partial_done_%=\n\t" - "veor.8 q0, q0, q0\n\t" - "mov r12, r11\n\t" - "vst1.32 {q0}, [r9]\n\t" - "cmp r12, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_aad_start_sw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_aad_start_dw_%=:\n\t" - "ldr r8, [r5], #4\n\t" - "sub r12, r12, #4\n\t" - "str r8, [r9], #4\n\t" - "cmp r12, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_aad_start_dw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_aad_start_sw_%=:\n\t" - "cmp r12, #2\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_aad_start_byte_%=\n\t" - "ldrh r8, [r5], #2\n\t" - "sub r12, r12, #2\n\t" - "strh r8, [r9], #2\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_aad_start_byte_%=:\n\t" - "cmp r12, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_aad_end_bytes_%=\n\t" - "ldrb r8, [r5], #1\n\t" - "subs r12, r12, #1\n\t" - "strb r8, [r9], #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_aad_start_byte_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_aad_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "vld1.32 {q14}, [r9]\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q7, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "\n" - "L_aes_gcm_decrypt_arm32_crypto_aad_partial_done_%=:\n\t" - /* in */ - "lsr r10, %[sz], #4\n\t" - "cmp r10, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_in_done_%=\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_in_start_1_%=\n\t" - "cmp r10, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_in_start_2_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_in_start_4_%=:\n\t" - "vldm %[in]!, {q0-q2}\n\t" - "vmov.i8 q12, #0x55\n\t" - "vmov.i8 q5, #51\n\t" - "vshl.u8 q14, q0, #1\n\t" - "vshl.u8 q15, q1, #1\n\t" - "vshl.u8 q4, q2, #1\n\t" - "ldr lr, [%[in]], #4\n\t" - "vshr.u8 q0, q0, #1\n\t" - "vshr.u8 q1, q1, #1\n\t" - "vshr.u8 q2, q2, #1\n\t" - "ldr r4, [%[in]], #4\n\t" - "vbit.8 q14, q0, q12\n\t" - "vbit.8 q15, q1, q12\n\t" - "vbit.8 q4, q2, q12\n\t" - "ldr r8, [%[in]], #4\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshl.u8 q1, q15, #2\n\t" - "vshl.u8 q2, q4, #2\n\t" - "ldr r12, [%[in]], #4\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vshr.u8 q15, q15, #2\n\t" - "vshr.u8 q4, q4, #2\n\t" - "rbit lr, lr\n\t" - "vbit.8 q0, q14, q5\n\t" - "rbit r4, r4\n\t" - "vbit.8 q1, q15, q5\n\t" - "rbit r8, r8\n\t" - "vbit.8 q2, q4, q5\n\t" - "rbit r12, r12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "rev lr, lr\n\t" - "vshl.u8 q15, q1, #4\n\t" - "rev r4, r4\n\t" - "vshl.u8 q4, q2, #4\n\t" - "rev r8, r8\n\t" - "vsri.u8 q14, q0, #4\n\t" - "rev r12, r12\n\t" - "vsri.u8 q15, q1, #4\n\t" - "vsri.u8 q4, q2, #4\n\t" - "vmov d10, lr, r4\n\t" - "vmov d11, r8, r12\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d11, d16\n\t" - "vmull.p64 q7, d10, d17\n\t" - "vmull.p64 q0, d10, d16\n\t" - "vmull.p64 q1, d11, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^2 */ - "vmull.p64 q3, d8, d18\n\t" - "vmull.p64 q7, d9, d19\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d9, d18\n\t" - "vmull.p64 q7, d8, d19\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^3 */ - "vmull.p64 q3, d30, d20\n\t" - "vmull.p64 q7, d31, d21\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d31, d20\n\t" - "vmull.p64 q7, d30, d21\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^4 */ - "vmull.p64 q3, d24, d22\n\t" - "vmull.p64 q7, d25, d23\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d25, d22\n\t" - "vmull.p64 q7, d24, d23\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "sub r10, r10, #4\n\t" - "cmp r10, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_in_start_4_%=\n\t" - "cmp r10, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_in_done_%=\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_in_start_1_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_in_start_2_%=:\n\t" - "vld1.32 {q14-q15}, [%[in]]!\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshl.u8 q1, q15, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vshr.u8 q15, q15, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vbif.8 q15, q1, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshl.u8 q1, q15, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vshr.u8 q15, q15, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vbit.8 q1, q15, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vshl.u8 q15, q1, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "vsri.u8 q15, q1, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d31, d16\n\t" - "vmull.p64 q7, d30, d17\n\t" - "vmull.p64 q0, d30, d16\n\t" - "vmull.p64 q1, d31, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* X += C * H^2 */ - "vmull.p64 q3, d24, d18\n\t" - "vmull.p64 q7, d25, d19\n\t" - "veor.8 q0, q0, q3\n\t" - "veor.8 q1, q1, q7\n\t" - "vmull.p64 q3, d25, d18\n\t" - "vmull.p64 q7, d24, d19\n\t" - "veor.8 q2, q2, q3\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "sub r10, r10, #2\n\t" - "cmp r10, #0\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_in_done_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_in_start_1_%=:\n\t" - "vld1.32 {q14}, [%[in]]!\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q7, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "\n" - "L_aes_gcm_decrypt_arm32_crypto_in_done_%=:\n\t" - "ands r11, %[sz], #15\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_in_partial_done_%=\n\t" - "veor.8 q0, q0, q0\n\t" - "mov r12, r11\n\t" - "vst1.32 {q0}, [r9]\n\t" - "cmp r12, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_in_start_sw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_in_start_dw_%=:\n\t" - "ldr r8, [%[in]], #4\n\t" - "sub r12, r12, #4\n\t" - "str r8, [r9], #4\n\t" - "cmp r12, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_in_start_dw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_in_start_sw_%=:\n\t" - "cmp r12, #2\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_in_start_byte_%=\n\t" - "ldrh r8, [%[in]], #2\n\t" - "sub r12, r12, #2\n\t" - "strh r8, [r9], #2\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_in_start_byte_%=:\n\t" - "cmp r12, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_in_end_bytes_%=\n\t" - "ldrb r8, [%[in]], #1\n\t" - "subs r12, r12, #1\n\t" - "strb r8, [r9], #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_in_start_byte_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_in_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "vld1.32 {q14}, [r9]\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q7, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q7, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q7\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - /* Done GHASH */ - "\n" - "L_aes_gcm_decrypt_arm32_crypto_in_partial_done_%=:\n\t" - "sub %[in], %[in], %[sz]\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_done_gcm_%=:\n\t" - /* nonceSz */ - "ldr r12, [sp]\n\t" - /* Load Nonce */ - "cmp r12, #12\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_ghash_nonce_%=\n\t" - "ldr r5, [%[nonce]]\n\t" - "ldr r8, [%[nonce], #4]\n\t" - "ldr r12, [%[nonce], #8]\n\t" - "vmov.i32 q6, #0x1000000\n\t" - "vmov.32 s24, r5\n\t" - "vmov.32 s25, r8\n\t" - "vmov.32 s26, r12\n\t" - "mov r5, #1\n\t" - "b L_aes_gcm_decrypt_arm32_crypto_done_nonce_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_ghash_nonce_%=:\n\t" - "lsr r10, r12, #4\n\t" - "cmp r10, #0\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_nonce_done_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_nonce_start_1_%=:\n\t" - "vld1.32 {q14}, [%[nonce]]!\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q6, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q6, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q6\n\t" - /* Reduce */ - "vmull.p64 q6, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d13\n\t" - "veor.8 d1, d1, d12\n\t" - "vmull.p64 q6, d2, d26\n\t" - "veor.8 q6, q6, q0\n\t" - /* Done GHASH */ - "subs r10, r10, #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_nonce_start_1_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_nonce_done_%=:\n\t" - "ands r11, r12, #15\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_nonce_partial_done_%=\n\t" - "veor.8 q0, q0, q0\n\t" - "mov r12, r11\n\t" - "vst1.32 {q0}, [r9]\n\t" - "cmp r12, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_nonce_start_sw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_nonce_start_dw_%=:\n\t" - "ldr r8, [%[nonce]], #4\n\t" - "sub r12, r12, #4\n\t" - "str r8, [r9], #4\n\t" - "cmp r12, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_nonce_start_dw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_nonce_start_sw_%=:\n\t" - "cmp r12, #2\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte_%=\n\t" - "ldrh r8, [%[nonce]], #2\n\t" - "sub r12, r12, #2\n\t" - "strh r8, [r9], #2\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte_%=:\n\t" - "cmp r12, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_nonce_end_bytes_%=\n\t" - "ldrb r8, [%[nonce]], #1\n\t" - "subs r12, r12, #1\n\t" - "strb r8, [r9], #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_nonce_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "vld1.32 {q14}, [r9]\n\t" - "vmov.i8 q12, #0x55\n\t" - "vshl.u8 q0, q14, #1\n\t" - "vshr.u8 q14, q14, #1\n\t" - "vbif.8 q14, q0, q12\n\t" - "vmov.i8 q12, #51\n\t" - "vshl.u8 q0, q14, #2\n\t" - "vshr.u8 q14, q14, #2\n\t" - "vbit.8 q0, q14, q12\n\t" - "vshl.u8 q14, q0, #4\n\t" - "vsri.u8 q14, q0, #4\n\t" - "veor.8 q12, q6, q14\n\t" - /* X = C * H^1 */ - "vmull.p64 q2, d25, d16\n\t" - "vmull.p64 q6, d24, d17\n\t" - "vmull.p64 q0, d24, d16\n\t" - "vmull.p64 q1, d25, d17\n\t" - "veor.8 q2, q2, q6\n\t" - /* Reduce */ - "vmull.p64 q6, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d13\n\t" - "veor.8 d1, d1, d12\n\t" - "vmull.p64 q6, d2, d26\n\t" - "veor.8 q6, q6, q0\n\t" - /* Done GHASH */ - "\n" - "L_aes_gcm_decrypt_arm32_crypto_nonce_partial_done_%=:\n\t" - "veor.8 q0, q0, q0\n\t" - /* nonceSz */ - "ldr r12, [sp]\n\t" - "lsr r10, r12, #29\n\t" - "lsl r11, r12, #3\n\t" - "rbit r10, r10\n\t" - "rbit r11, r11\n\t" - "vmov.32 s2, r10\n\t" - "vmov.32 s3, r11\n\t" - "veor.8 q6, q6, q0\n\t" - "vmull.p64 q2, d13, d16\n\t" - "vmull.p64 q12, d12, d17\n\t" - "vmull.p64 q0, d12, d16\n\t" - "vmull.p64 q1, d13, d17\n\t" - "veor.8 q2, q2, q12\n\t" - "vmull.p64 q6, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d13\n\t" - "veor.8 d1, d1, d12\n\t" - "vmull.p64 q6, d2, d26\n\t" - "veor.8 q6, q6, q0\n\t" - "vmov.32 r5, s24\n\t" - "vmov.32 r8, s25\n\t" - "vmov.32 r12, s26\n\t" - "rbit r5, r5\n\t" - "rbit r8, r8\n\t" - "rbit r12, r12\n\t" - "rev r5, r5\n\t" - "rev r8, r8\n\t" - "rev r12, r12\n\t" - "vmov.32 s24, r5\n\t" - "vmov.32 s25, r8\n\t" - "vmov.32 s26, r12\n\t" - "vmov.32 r5, s27\n\t" - "rbit r5, r5\n\t" - "rev r5, r5\n\t" - "vmov.32 s27, r5\n\t" - "rev r5, r5\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_done_nonce_%=:\n\t" - /* reg */ - "ldr r9, [sp, #32]\n\t" - "vst1.32 {q7}, [r9]\n\t" - /* tmp */ - "ldr r9, [sp, #28]\n\t" - "vldm.32 r7!, {q0-q3}\n\t" - "vldm.32 r7!, {q7-q13}\n\t" - /* nr */ - "ldr r12, [sp, #36]\n\t" - "lsr r10, %[sz], #4\n\t" - "and r11, %[sz], #15\n\t" - "cmp r12, #12\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_start_128_%=\n\t" - "bgt L_aes_gcm_decrypt_arm32_crypto_start_256_%=\n\t" - /* AES_GCM_192 */ -#ifndef NO_AES_192 - "cmp r10, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_192_done_%=\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_192_start_1_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_start_2_%=:\n\t" - "add r8, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "add r5, r5, #2\n\t" - "vmov.8 q5, q6\n\t" - "rev r8, r8\n\t" - "rev r12, r5\n\t" - "vmov s19, r8\n\t" - "vmov s23, r12\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q0\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q1\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q2\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q3\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q7\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q8\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q9\n\t" - "aesmc.8 q5, q5\n\t" - "subs r10, r10, #2\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q10\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q11\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q12\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q13\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "aese.8 q5, q14\n\t" - "veor.8 q5, q5, q15\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "sub r7, r7, #16\n\t" - "veor.8 q14, q14, q4\n\t" - "veor.8 q15, q15, q5\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "cmp r10, #1\n\t" - "bgt L_aes_gcm_decrypt_arm32_crypto_192_start_2_%=\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_192_done_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_start_1_%=:\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "sub r7, r7, #16\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [%[out]]!\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_done_%=:\n\t" - "ands r11, %[sz], #15\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_192_partial_done_%=\n\t" - "veor.8 q14, q14, q14\n\t" - "mov r4, r11\n\t" - "vst1.32 {q14}, [r9]\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_192_start_sw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_start_dw_%=:\n\t" - "ldr lr, [%[in]], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [r9], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_192_start_dw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_192_start_byte_%=\n\t" - "ldrh lr, [%[in]], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [r9], #2\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_192_end_bytes_%=\n\t" - "ldrb lr, [%[in]], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [r9], #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_192_start_byte_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "vld1.8 {q14}, [r9]\n\t" - "sub r7, r7, #16\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [r9]\n\t" - "mov r4, r11\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_192_out_start_sw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_out_start_dw_%=:\n\t" - "ldr lr, [r9], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [%[out]], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_192_out_start_dw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_out_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte_%=\n\t" - "ldrh lr, [r9], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [%[out]], #2\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_192_out_end_bytes_%=\n\t" - "ldrb lr, [r9], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [%[out]], #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_out_end_bytes_%=:\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_192_partial_done_%=:\n\t" - /* Finish */ - "add r8, %[sz], #15\n\t" - "sub r8, r5, r8, lsr #4\n\t" - "rev r8, r8\n\t" - "vmov.32 s27, r8\n\t" - "aese.8 q6, q0\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q1\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q2\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q3\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q7\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q8\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q9\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q10\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q11\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q12\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q6, q13\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q6, q14\n\t" - "veor.8 q6, q6, q15\n\t" - "sub r7, r7, #16\n\t" -#endif /* !NO_AES_192 */ - "b L_aes_gcm_decrypt_arm32_crypto_done_enc_%=\n\t" - /* AES_GCM_256 */ - "\n" - "L_aes_gcm_decrypt_arm32_crypto_start_256_%=:\n\t" -#ifndef NO_AES_256 - "cmp r10, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_256_done_%=\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_256_start_1_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_start_2_%=:\n\t" - "add r8, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "add r5, r5, #2\n\t" - "vmov.8 q5, q6\n\t" - "rev r8, r8\n\t" - "rev r12, r5\n\t" - "vmov s19, r8\n\t" - "vmov s23, r12\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q0\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q1\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q2\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q3\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q7\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q8\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q9\n\t" - "aesmc.8 q5, q5\n\t" - "subs r10, r10, #2\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q10\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q11\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q12\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q13\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q15}, [r7]!\n\t" - "aese.8 q4, q14\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q14\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q15\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q15\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "aese.8 q5, q14\n\t" - "veor.8 q5, q5, q15\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "sub r7, r7, #48\n\t" - "veor.8 q14, q14, q4\n\t" - "veor.8 q15, q15, q5\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "cmp r10, #1\n\t" - "bgt L_aes_gcm_decrypt_arm32_crypto_256_start_2_%=\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_256_done_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_start_1_%=:\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]!\n\t" - "aese.8 q4, q14\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q15\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "sub r7, r7, #48\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [%[out]]!\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_done_%=:\n\t" - "ands r11, %[sz], #15\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_256_partial_done_%=\n\t" - "veor.8 q14, q14, q14\n\t" - "mov r4, r11\n\t" - "vst1.32 {q14}, [r9]\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_256_start_sw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_start_dw_%=:\n\t" - "ldr lr, [%[in]], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [r9], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_256_start_dw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_256_start_byte_%=\n\t" - "ldrh lr, [%[in]], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [r9], #2\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_256_end_bytes_%=\n\t" - "ldrb lr, [%[in]], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [r9], #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_256_start_byte_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q12\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q13\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]!\n\t" - "aese.8 q4, q14\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q4, q15\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q4, q14\n\t" - "veor.8 q4, q4, q15\n\t" - "vld1.8 {q14}, [r9]\n\t" - "sub r7, r7, #48\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [r9]\n\t" - "mov r4, r11\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_256_out_start_sw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_out_start_dw_%=:\n\t" - "ldr lr, [r9], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [%[out]], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_256_out_start_dw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_out_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte_%=\n\t" - "ldrh lr, [r9], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [%[out]], #2\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_256_out_end_bytes_%=\n\t" - "ldrb lr, [r9], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [%[out]], #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_out_end_bytes_%=:\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_256_partial_done_%=:\n\t" - /* Finish */ - "add r8, %[sz], #15\n\t" - "sub r8, r5, r8, lsr #4\n\t" - "rev r8, r8\n\t" - "vmov.32 s27, r8\n\t" - "aese.8 q6, q0\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q1\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q2\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q3\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q7\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q8\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q9\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q10\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q11\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q12\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q6, q13\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q15}, [r7]!\n\t" - "aese.8 q6, q14\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q14}, [r7]!\n\t" - "aese.8 q6, q15\n\t" - "aesmc.8 q6, q6\n\t" - "vld1.32 {q15}, [r7]\n\t" - "aese.8 q6, q14\n\t" - "veor.8 q6, q6, q15\n\t" - "sub r7, r7, #48\n\t" -#endif /* !NO_AES_256 */ - "b L_aes_gcm_decrypt_arm32_crypto_done_enc_%=\n\t" - /* AES_GCM_128 */ - "\n" - "L_aes_gcm_decrypt_arm32_crypto_start_128_%=:\n\t" -#ifndef NO_AES_128 - "cmp r10, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_128_done_%=\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_128_start_1_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_start_2_%=:\n\t" - "add r8, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "add r5, r5, #2\n\t" - "vmov.8 q5, q6\n\t" - "rev r8, r8\n\t" - "rev r12, r5\n\t" - "vmov s19, r8\n\t" - "vmov s23, r12\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q0\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q1\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q2\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q3\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q7\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q8\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q9\n\t" - "aesmc.8 q5, q5\n\t" - "subs r10, r10, #2\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q10\n\t" - "aesmc.8 q5, q5\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q5, q11\n\t" - "aesmc.8 q5, q5\n\t" - "vld1.8 {q14-q15}, [%[in]]!\n\t" - "aese.8 q4, q12\n\t" - "veor.8 q4, q4, q13\n\t" - "aese.8 q5, q12\n\t" - "veor.8 q5, q5, q13\n\t" - "veor.8 q14, q14, q4\n\t" - "veor.8 q15, q15, q5\n\t" - "vst1.8 {q14-q15}, [%[out]]!\n\t" - "cmp r10, #1\n\t" - "bgt L_aes_gcm_decrypt_arm32_crypto_128_start_2_%=\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_128_done_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_start_1_%=:\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.8 {q14}, [%[in]]!\n\t" - "aese.8 q4, q12\n\t" - "veor.8 q4, q4, q13\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [%[out]]!\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_done_%=:\n\t" - "ands r11, %[sz], #15\n\t" - "beq L_aes_gcm_decrypt_arm32_crypto_128_partial_done_%=\n\t" - "veor.8 q14, q14, q14\n\t" - "mov r4, r11\n\t" - "vst1.32 {q14}, [r9]\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_128_start_sw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_start_dw_%=:\n\t" - "ldr lr, [%[in]], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [r9], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_128_start_dw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_128_start_byte_%=\n\t" - "ldrh lr, [%[in]], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [r9], #2\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_128_end_bytes_%=\n\t" - "ldrb lr, [%[in]], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [r9], #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_128_start_byte_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_end_bytes_%=:\n\t" - "sub r9, r9, r11\n\t" - "add r5, r5, #1\n\t" - "vmov.8 q4, q6\n\t" - "rev r8, r5\n\t" - "vmov s19, r8\n\t" - "aese.8 q4, q0\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q1\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q2\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q3\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q7\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q8\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q9\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q10\n\t" - "aesmc.8 q4, q4\n\t" - "aese.8 q4, q11\n\t" - "aesmc.8 q4, q4\n\t" - "vld1.8 {q14}, [r9]\n\t" - "aese.8 q4, q12\n\t" - "veor.8 q4, q4, q13\n\t" - "veor.8 q14, q14, q4\n\t" - "vst1.32 {q14}, [r9]\n\t" - "mov r4, r11\n\t" - "cmp r4, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_128_out_start_sw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_out_start_dw_%=:\n\t" - "ldr lr, [r9], #4\n\t" - "sub r4, r4, #4\n\t" - "str lr, [%[out]], #4\n\t" - "cmp r4, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_128_out_start_dw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_out_start_sw_%=:\n\t" - "cmp r4, #2\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte_%=\n\t" - "ldrh lr, [r9], #2\n\t" - "sub r4, r4, #2\n\t" - "strh lr, [%[out]], #2\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte_%=:\n\t" - "cmp r4, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_128_out_end_bytes_%=\n\t" - "ldrb lr, [r9], #1\n\t" - "subs r4, r4, #1\n\t" - "strb lr, [%[out]], #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_out_end_bytes_%=:\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_128_partial_done_%=:\n\t" - /* Finish */ - "add r8, %[sz], #15\n\t" - "sub r8, r5, r8, lsr #4\n\t" - "rev r8, r8\n\t" - "vmov.32 s27, r8\n\t" - "aese.8 q6, q0\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q1\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q2\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q3\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q7\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q8\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q9\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q10\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q11\n\t" - "aesmc.8 q6, q6\n\t" - "aese.8 q6, q12\n\t" - "veor.8 q6, q6, q13\n\t" -#endif /* !NO_AES_128 */ - "\n" - "L_aes_gcm_decrypt_arm32_crypto_done_enc_%=:\n\t" - "vmov.i8 q13, #0x87\n\t" - "vshr.u64 q13, q13, #56\n\t" - /* gcm_h */ - "ldr r8, [sp, #24]\n\t" - "vld1.32 {q8}, [r8]\n\t" - /* reg */ - "ldr r9, [sp, #32]\n\t" - "vld1.32 {q7}, [r9]\n\t" - /* tmp */ - "ldr r9, [sp, #28]\n\t" - "lsr lr, r6, #29\n\t" - "lsl r6, r6, #3\n\t" - "rbit lr, lr\n\t" - "rbit r6, r6\n\t" - "vmov s0, lr\n\t" - "vmov s1, r6\n\t" - "lsr lr, %[sz], #29\n\t" - "lsl %[sz], %[sz], #3\n\t" - "rbit lr, lr\n\t" - "rbit %[sz], %[sz]\n\t" - "vmov s2, lr\n\t" - "vmov s3, %[sz]\n\t" - "veor.8 q7, q7, q0\n\t" - "vmull.p64 q2, d15, d16\n\t" - "vmull.p64 q3, d14, d17\n\t" - "vmull.p64 q0, d14, d16\n\t" - "vmull.p64 q1, d15, d17\n\t" - "veor.8 q2, q2, q3\n\t" - /* Reduce */ - "vmull.p64 q7, d3, d27\n\t" - "veor.8 d2, d2, d5\n\t" - "veor.8 d1, d1, d4\n\t" - "veor.8 d2, d2, d15\n\t" - "veor.8 d1, d1, d14\n\t" - "vmull.p64 q7, d2, d26\n\t" - "veor.8 q7, q7, q0\n\t" - "vmov.i8 q0, #0x55\n\t" - "vshl.u8 q1, q7, #1\n\t" - "vshr.u8 q7, q7, #1\n\t" - "vbif.8 q7, q1, q0\n\t" - "vmov.i8 q0, #51\n\t" - "vshl.u8 q1, q7, #2\n\t" - "vshr.u8 q7, q7, #2\n\t" - "vbit.8 q1, q7, q0\n\t" - "vshl.u8 q7, q1, #4\n\t" - "vsri.u8 q7, q1, #4\n\t" - /* DONE */ - "veor.8 q7, q7, q6\n\t" - /* tag */ - "ldr lr, [sp, #4]\n\t" - /* tagSz */ - "ldr r4, [sp, #8]\n\t" - "cmp r4, #16\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_tag_part_tag_%=\n\t" - "vld1.8 {q0}, [lr]\n\t" - "b L_aes_gcm_decrypt_arm32_crypto_tag_tag_loaded_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_tag_part_tag_%=:\n\t" - "veor.8 q0, q0, q0\n\t" - "mov r12, r4\n\t" - "vst1.32 {q0}, [r9]\n\t" - "cmp r12, #4\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_sw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_dw_%=:\n\t" - "ldr r8, [lr], #4\n\t" - "sub r12, r12, #4\n\t" - "str r8, [r9], #4\n\t" - "cmp r12, #4\n\t" - "bge L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_dw_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_sw_%=:\n\t" - "cmp r12, #2\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte_%=\n\t" - "ldrh r8, [lr], #2\n\t" - "sub r12, r12, #2\n\t" - "strh r8, [r9], #2\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte_%=:\n\t" - "cmp r12, #1\n\t" - "blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_end_bytes_%=\n\t" - "ldrb r8, [lr], #1\n\t" - "subs r12, r12, #1\n\t" - "strb r8, [r9], #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte_%=\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_tag_tag_end_bytes_%=:\n\t" - "sub r9, r9, r4\n\t" - "vld1.32 {q0}, [r9]\n\t" - "mov r12, #16\n\t" - "vst1.32 {q7}, [r9]\n\t" - "sub r12, r12, r4\n\t" - "eor r8, r8, r8\n\t" - "add r9, r9, r4\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_tag_calc_tag_byte_%=:\n\t" - "strb r8, [r9], #1\n\t" - "subs r12, r12, #1\n\t" - "bne L_aes_gcm_decrypt_arm32_crypto_tag_calc_tag_byte_%=\n\t" - "subs r9, r9, #16\n\t" - "vld1.32 {q7}, [r9]\n\t" - "\n" - "L_aes_gcm_decrypt_arm32_crypto_tag_tag_loaded_%=:\n\t" - "vceq.i32 q0, q0, q7\n\t" - "vmov r5, s0\n\t" - "vmov r8, s1\n\t" - "and r12, r5, r8\n\t" - "vmov r5, s2\n\t" - "vmov r8, s3\n\t" - "and r12, r12, r5\n\t" - "and r12, r12, r8\n\t" - "mov r5, #-180\n\t" - "mvn r12, r12\n\t" - "and %[in], r5, r12\n\t" - "pop {%[nonceSz], %[tag]}\n\t" - "pop {%[tagSz], %[aad], %[aadSz], %[key], %[gcm_h], %[tmp], %[reg], %[nr]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), - [nonce] "+r" (nonce), [nonceSz] "+r" (nonceSz), [tag] "+r" (tag), - [tagSz] "+r" (tagSz), [aad] "+r" (aad), [aadSz] "+r" (aadSz), - [key] "+r" (key), [gcm_h] "+r" (gcm_h), [tmp] "+r" (tmp), - [reg] "+r" (reg), [nr] "+r" (nr) - : -#else - : - : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [nonce] "r" (nonce), - [nonceSz] "r" (nonceSz), [tag] "r" (tag), [tagSz] "r" (tagSz), - [aad] "r" (aad), [aadSz] "r" (aadSz), [key] "r" (key), - [gcm_h] "r" (gcm_h), [tmp] "r" (tmp), [reg] "r" (reg), [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", - "q9", "q10", "q11", "q12", "q13", "q14", "q15" - ); - return (word32)(size_t)in; -} - -#endif /* HAVE_AES_DECRYPT */ -#endif /* HAVE_AESGCM */ -#ifdef WOLFSSL_AES_XTS -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_XTS_encrypt_AARCH32(const byte* in_p, - byte* out_p, word32 sz_p, const byte* i_p, byte* key_p, byte* key2_p, - byte* tmp_p, int nr_p) -#else -WC_OMIT_FRAME_POINTER void AES_XTS_encrypt_AARCH32(const byte* in, byte* out, - word32 sz, const byte* i, byte* key, byte* key2, byte* tmp, int nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* in __asm__ ("r0") = (const byte*)in_p; - register byte* out __asm__ ("r1") = (byte*)out_p; - register word32 sz __asm__ ("r2") = (word32)sz_p; - register const byte* i __asm__ ("r3") = (const byte*)i_p; - register byte* key __asm__ ("r12") = (byte*)key_p; - register byte* key2 __asm__ ("lr") = (byte*)key2_p; - register byte* tmp __asm__ ("r4") = (byte*)tmp_p; - register int nr __asm__ ("r5") = (int)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[tmp], %[nr]}\n\t" - "push {%[key], %[key2]}\n\t" - "ldr r12, [sp]\n\t" - "ldr lr, [sp, #4]\n\t" - "ldr r5, [sp, #12]\n\t" - "vldm.32 lr!, {q2-q9}\n\t" - "lsr r4, %[sz], #4\n\t" - "and %[sz], %[sz], #15\n\t" - "vld1.8 {q0}, [%[i]]\n\t" - "cmp r5, #12\n\t" - "blt L_aes_xts_encrypt_arm32_crypto_start_128_%=\n\t" - "bgt L_aes_xts_encrypt_arm32_crypto_start_256_%=\n\t" - /* AES_XTS_192 */ -#ifndef NO_AES_192 - "vldm.32 lr!, {q10-q14}\n\t" - "aese.8 q0, q2\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q11\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q12\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q13\n\t" - "veor.32 q0, q0, q14\n\t" - "vmov r6, r7, d0\n\t" - "vmov r8, r9, d1\n\t" - "vldm.32 r12!, {q2-q9}\n\t" - "vldm.32 r12!, {q10-q14}\n\t" - "mov lr, #0x87\n\t" - "cmp r4, #1\n\t" - "blt L_aes_xts_encrypt_arm32_crypto_192_done_%=\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_192_start_1_%=:\n\t" - "vld1.8 {q1}, [%[in]]!\n\t" - "veor.32 q1, q1, q0\n\t" - "aese.8 q1, q2\n\t" - "aesmc.8 q1, q1\n\t" - "and r5, lr, r9, asr #31\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "lsl r9, r9, #1\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "orr r9, r9, r8, lsr #31\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "lsl r8, r8, #1\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "orr r8, r8, r7, lsr #31\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "lsl r7, r7, #1\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "orr r7, r7, r6, lsr #31\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "eor r6, r5, r6, lsl #1\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q11\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q12\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q13\n\t" - "veor.32 q1, q1, q14\n\t" - "veor.32 q1, q1, q0\n\t" - "vmov d0, r6, r7\n\t" - "vmov d1, r8, r9\n\t" - "subs r4, r4, #1\n\t" - "vst1.8 {q1}, [%[out]]!\n\t" - "bne L_aes_xts_encrypt_arm32_crypto_192_start_1_%=\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_192_done_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_xts_encrypt_arm32_crypto_192_partial_done_%=\n\t" - "sub %[out], %[out], #16\n\t" - "ldr r4, [sp, #8]\n\t" - "vld1.8 {q1}, [%[out]]!\n\t" - "vst1.32 {q1}, [r4]\n\t" - "mov r5, %[sz]\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_192_start_byte_%=:\n\t" - "ldrb r8, [r4]\n\t" - "ldrb r9, [%[in]], #1\n\t" - "strb r8, [%[out]], #1\n\t" - "strb r9, [r4], #1\n\t" - "subs r5, r5, #1\n\t" - "bgt L_aes_xts_encrypt_arm32_crypto_192_start_byte_%=\n\t" - "sub %[out], %[out], %[sz]\n\t" - "sub r4, r4, %[sz]\n\t" - "sub %[out], %[out], #16\n\t" - "vld1.32 {q1}, [r4]\n\t" - "veor.32 q1, q1, q0\n\t" - "aese.8 q1, q2\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q11\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q12\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q13\n\t" - "veor.32 q1, q1, q14\n\t" - "veor.32 q1, q1, q0\n\t" - "vst1.8 {q1}, [%[out]]\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_192_partial_done_%=:\n\t" -#endif /* !NO_AES_192 */ - "b L_aes_xts_encrypt_arm32_crypto_done_%=\n\t" - /* AES_XTS_256 */ - "\n" - "L_aes_xts_encrypt_arm32_crypto_start_256_%=:\n\t" -#ifndef NO_AES_256 - "vldm.32 lr!, {q10-q13}\n\t" - "aese.8 q0, q2\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q11\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q12\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q14}, [lr]!\n\t" - "aese.8 q0, q13\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q15}, [lr]!\n\t" - "aese.8 q0, q14\n\t" - "aesmc.8 q0, q0\n\t" - "vld1.32 {q14}, [lr]!\n\t" - "aese.8 q0, q15\n\t" - "veor.32 q0, q0, q14\n\t" - "vmov r6, r7, d0\n\t" - "vmov r8, r9, d1\n\t" - "vldm.32 r12!, {q2-q9}\n\t" - "vldm.32 r12!, {q10-q13}\n\t" - "mov lr, #0x87\n\t" - "cmp r4, #1\n\t" - "blt L_aes_xts_encrypt_arm32_crypto_256_done_%=\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_256_start_1_%=:\n\t" - "vld1.8 {q1}, [%[in]]!\n\t" - "veor.32 q1, q1, q0\n\t" - "aese.8 q1, q2\n\t" - "aesmc.8 q1, q1\n\t" - "and r5, lr, r9, asr #31\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "lsl r9, r9, #1\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "orr r9, r9, r8, lsr #31\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "lsl r8, r8, #1\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "orr r8, r8, r7, lsr #31\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "lsl r7, r7, #1\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "orr r7, r7, r6, lsr #31\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "eor r6, r5, r6, lsl #1\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q11\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q12\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aese.8 q1, q13\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q15}, [r12]!\n\t" - "aese.8 q1, q14\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aese.8 q1, q15\n\t" - "veor.32 q1, q1, q14\n\t" - "sub r12, r12, #48\n\t" - "veor.32 q1, q1, q0\n\t" - "vmov d0, r6, r7\n\t" - "vmov d1, r8, r9\n\t" - "subs r4, r4, #1\n\t" - "vst1.8 {q1}, [%[out]]!\n\t" - "bne L_aes_xts_encrypt_arm32_crypto_256_start_1_%=\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_256_done_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_xts_encrypt_arm32_crypto_256_partial_done_%=\n\t" - "sub %[out], %[out], #16\n\t" - "ldr r4, [sp, #8]\n\t" - "vld1.8 {q1}, [%[out]]!\n\t" - "vst1.32 {q1}, [r4]\n\t" - "mov r5, %[sz]\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_256_start_byte_%=:\n\t" - "ldrb r8, [r4]\n\t" - "ldrb r9, [%[in]], #1\n\t" - "strb r8, [%[out]], #1\n\t" - "strb r9, [r4], #1\n\t" - "subs r5, r5, #1\n\t" - "bgt L_aes_xts_encrypt_arm32_crypto_256_start_byte_%=\n\t" - "sub %[out], %[out], %[sz]\n\t" - "sub r4, r4, %[sz]\n\t" - "sub %[out], %[out], #16\n\t" - "vld1.32 {q1}, [r4]\n\t" - "veor.32 q1, q1, q0\n\t" - "aese.8 q1, q2\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q11\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q12\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aese.8 q1, q13\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q15}, [r12]!\n\t" - "aese.8 q1, q14\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aese.8 q1, q15\n\t" - "veor.32 q1, q1, q14\n\t" - "veor.32 q1, q1, q0\n\t" - "vst1.8 {q1}, [%[out]]\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_256_partial_done_%=:\n\t" -#endif /* !NO_AES_256 */ - "b L_aes_xts_encrypt_arm32_crypto_done_%=\n\t" - /* AES_XTS_128 */ - "\n" - "L_aes_xts_encrypt_arm32_crypto_start_128_%=:\n\t" -#ifndef NO_AES_128 - "vldm.32 lr!, {q10-q12}\n\t" - "aese.8 q0, q2\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q3\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q4\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q5\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q6\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q7\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q8\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q9\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q10\n\t" - "aesmc.8 q0, q0\n\t" - "aese.8 q0, q11\n\t" - "veor.32 q0, q0, q12\n\t" - "vmov r6, r7, d0\n\t" - "vmov r8, r9, d1\n\t" - "vldm.32 r12!, {q2-q9}\n\t" - "vldm.32 r12!, {q10-q12}\n\t" - "mov lr, #0x87\n\t" - "cmp r4, #1\n\t" - "blt L_aes_xts_encrypt_arm32_crypto_128_done_%=\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_128_start_1_%=:\n\t" - "vld1.8 {q1}, [%[in]]!\n\t" - "veor.32 q1, q1, q0\n\t" - "aese.8 q1, q2\n\t" - "aesmc.8 q1, q1\n\t" - "and r5, lr, r9, asr #31\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "lsl r9, r9, #1\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "orr r9, r9, r8, lsr #31\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "lsl r8, r8, #1\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "orr r8, r8, r7, lsr #31\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "lsl r7, r7, #1\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "orr r7, r7, r6, lsr #31\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "eor r6, r5, r6, lsl #1\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q11\n\t" - "veor.32 q1, q1, q12\n\t" - "veor.32 q1, q1, q0\n\t" - "vmov d0, r6, r7\n\t" - "vmov d1, r8, r9\n\t" - "subs r4, r4, #1\n\t" - "vst1.8 {q1}, [%[out]]!\n\t" - "bne L_aes_xts_encrypt_arm32_crypto_128_start_1_%=\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_128_done_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_xts_encrypt_arm32_crypto_128_partial_done_%=\n\t" - "sub %[out], %[out], #16\n\t" - "ldr r4, [sp, #8]\n\t" - "vld1.8 {q1}, [%[out]]!\n\t" - "vst1.32 {q1}, [r4]\n\t" - "mov r5, %[sz]\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_128_start_byte_%=:\n\t" - "ldrb r8, [r4]\n\t" - "ldrb r9, [%[in]], #1\n\t" - "strb r8, [%[out]], #1\n\t" - "strb r9, [r4], #1\n\t" - "subs r5, r5, #1\n\t" - "bgt L_aes_xts_encrypt_arm32_crypto_128_start_byte_%=\n\t" - "sub %[out], %[out], %[sz]\n\t" - "sub r4, r4, %[sz]\n\t" - "sub %[out], %[out], #16\n\t" - "vld1.32 {q1}, [r4]\n\t" - "veor.32 q1, q1, q0\n\t" - "aese.8 q1, q2\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q11\n\t" - "veor.32 q1, q1, q12\n\t" - "veor.32 q1, q1, q0\n\t" - "vst1.8 {q1}, [%[out]]\n\t" - "\n" - "L_aes_xts_encrypt_arm32_crypto_128_partial_done_%=:\n\t" -#endif /* !NO_AES_128 */ - "\n" - "L_aes_xts_encrypt_arm32_crypto_done_%=:\n\t" - "pop {%[key], %[key2]}\n\t" - "pop {%[tmp], %[nr]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [i] "+r" (i), - [key] "+r" (key), [key2] "+r" (key2), [tmp] "+r" (tmp), - [nr] "+r" (nr) - : -#else - : - : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [i] "r" (i), - [key] "r" (key), [key2] "r" (key2), [tmp] "r" (tmp), [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r6", "r7", "r8", "r9", "q0", "q1", "q2", "q3", "q4", - "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", - "q15" - ); -} - -#ifdef HAVE_AES_DECRYPT -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_XTS_decrypt_AARCH32(const byte* in_p, - byte* out_p, word32 sz_p, const byte* i_p, byte* key_p, byte* key2_p, - byte* tmp_p, int nr_p) -#else -WC_OMIT_FRAME_POINTER void AES_XTS_decrypt_AARCH32(const byte* in, byte* out, - word32 sz, const byte* i, byte* key, byte* key2, byte* tmp, int nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const byte* in __asm__ ("r0") = (const byte*)in_p; - register byte* out __asm__ ("r1") = (byte*)out_p; - register word32 sz __asm__ ("r2") = (word32)sz_p; - register const byte* i __asm__ ("r3") = (const byte*)i_p; - register byte* key __asm__ ("r12") = (byte*)key_p; - register byte* key2 __asm__ ("lr") = (byte*)key2_p; - register byte* tmp __asm__ ("r4") = (byte*)tmp_p; - register int nr __asm__ ("r5") = (int)nr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[tmp], %[nr]}\n\t" - "push {%[key], %[key2]}\n\t" - "ldr r12, [sp]\n\t" - "ldr lr, [sp, #4]\n\t" - "vldm.32 lr!, {q2-q9}\n\t" - "eor r5, r5, r5\n\t" - "lsr r4, %[sz], #4\n\t" - "ands %[sz], %[sz], #15\n\t" - "sub r5, r5, %[sz]\n\t" - "sub r4, r4, r5, lsr #31\n\t" - "vld1.8 {q1}, [%[i]]\n\t" - "ldr r5, [sp, #12]\n\t" - "cmp r5, #12\n\t" - "blt L_aes_xts_decrypt_arm32_crypto_start_128_%=\n\t" - "bgt L_aes_xts_decrypt_arm32_crypto_start_256_%=\n\t" - /* AES_XTS_192 */ -#ifndef NO_AES_192 - "vldm.32 lr!, {q10-q14}\n\t" - "aese.8 q1, q2\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q11\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q12\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q13\n\t" - "veor.32 q1, q1, q14\n\t" - "vmov r6, r7, d2\n\t" - "vmov r8, r9, d3\n\t" - "vldm.32 r12!, {q2-q9}\n\t" - "vldm.32 r12!, {q10-q14}\n\t" - "mov lr, #0x87\n\t" - "cmp r4, #1\n\t" - "blt L_aes_xts_decrypt_arm32_crypto_192_done_%=\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_192_start_1_%=:\n\t" - "vld1.8 {q0}, [%[in]]!\n\t" - "veor.32 q0, q0, q1\n\t" - "aesd.8 q0, q2\n\t" - "aesimc.8 q0, q0\n\t" - "and r5, lr, r9, asr #31\n\t" - "aesd.8 q0, q3\n\t" - "aesimc.8 q0, q0\n\t" - "lsl r9, r9, #1\n\t" - "aesd.8 q0, q4\n\t" - "aesimc.8 q0, q0\n\t" - "orr r9, r9, r8, lsr #31\n\t" - "aesd.8 q0, q5\n\t" - "aesimc.8 q0, q0\n\t" - "lsl r8, r8, #1\n\t" - "aesd.8 q0, q6\n\t" - "aesimc.8 q0, q0\n\t" - "orr r8, r8, r7, lsr #31\n\t" - "aesd.8 q0, q7\n\t" - "aesimc.8 q0, q0\n\t" - "lsl r7, r7, #1\n\t" - "aesd.8 q0, q8\n\t" - "aesimc.8 q0, q0\n\t" - "orr r7, r7, r6, lsr #31\n\t" - "aesd.8 q0, q9\n\t" - "aesimc.8 q0, q0\n\t" - "eor r6, r5, r6, lsl #1\n\t" - "aesd.8 q0, q10\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q11\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q12\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q13\n\t" - "veor.32 q0, q0, q14\n\t" - "veor.32 q0, q0, q1\n\t" - "vmov d2, r6, r7\n\t" - "vmov d3, r8, r9\n\t" - "subs r4, r4, #1\n\t" - "vst1.8 {q0}, [%[out]]!\n\t" - "bne L_aes_xts_decrypt_arm32_crypto_192_start_1_%=\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_192_done_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_xts_decrypt_arm32_crypto_192_partial_done_%=\n\t" - "and r5, lr, r9, asr #31\n\t" - "lsl r9, r9, #1\n\t" - "orr r9, r9, r8, lsr #31\n\t" - "lsl r8, r8, #1\n\t" - "orr r8, r8, r7, lsr #31\n\t" - "lsl r7, r7, #1\n\t" - "orr r7, r7, r6, lsr #31\n\t" - "eor r6, r5, r6, lsl #1\n\t" - "vmov d30, r6, r7\n\t" - "vmov d31, r8, r9\n\t" - "vld1.8 {q0}, [%[in]]!\n\t" - "ldr r4, [sp, #8]\n\t" - "veor.32 q0, q0, q15\n\t" - "aesd.8 q0, q2\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q3\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q4\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q5\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q6\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q7\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q8\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q9\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q10\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q11\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q12\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q13\n\t" - "veor.32 q0, q0, q14\n\t" - "veor.32 q0, q0, q15\n\t" - "vst1.32 {q0}, [r4]\n\t" - "add %[out], %[out], #16\n\t" - "mov r5, %[sz]\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_192_start_byte_%=:\n\t" - "ldrb r8, [r4]\n\t" - "ldrb r9, [%[in]], #1\n\t" - "strb r8, [%[out]], #1\n\t" - "strb r9, [r4], #1\n\t" - "subs r5, r5, #1\n\t" - "bgt L_aes_xts_decrypt_arm32_crypto_192_start_byte_%=\n\t" - "sub %[out], %[out], %[sz]\n\t" - "sub r4, r4, %[sz]\n\t" - "sub %[out], %[out], #16\n\t" - "vld1.32 {q0}, [r4]\n\t" - "veor.32 q0, q0, q1\n\t" - "aesd.8 q0, q2\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q3\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q4\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q5\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q6\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q7\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q8\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q9\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q10\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q11\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q12\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q13\n\t" - "veor.32 q0, q0, q14\n\t" - "veor.32 q0, q0, q1\n\t" - "vst1.8 {q0}, [%[out]]\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_192_partial_done_%=:\n\t" -#endif /* !NO_AES_192 */ - "b L_aes_xts_decrypt_arm32_crypto_done_%=\n\t" - /* AES_XTS_256 */ - "\n" - "L_aes_xts_decrypt_arm32_crypto_start_256_%=:\n\t" -#ifndef NO_AES_256 - "vldm.32 lr!, {q10-q13}\n\t" - "aese.8 q1, q2\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q11\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q12\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q14}, [lr]!\n\t" - "aese.8 q1, q13\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q15}, [lr]!\n\t" - "aese.8 q1, q14\n\t" - "aesmc.8 q1, q1\n\t" - "vld1.32 {q14}, [lr]!\n\t" - "aese.8 q1, q15\n\t" - "veor.32 q1, q1, q14\n\t" - "vmov r6, r7, d2\n\t" - "vmov r8, r9, d3\n\t" - "vldm.32 r12!, {q2-q9}\n\t" - "vldm.32 r12!, {q10-q12}\n\t" - "mov lr, #0x87\n\t" - "cmp r4, #1\n\t" - "blt L_aes_xts_decrypt_arm32_crypto_256_done_%=\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_256_start_1_%=:\n\t" - "vld1.8 {q0}, [%[in]]!\n\t" - "veor.32 q0, q0, q1\n\t" - "aesd.8 q0, q2\n\t" - "aesimc.8 q0, q0\n\t" - "and r5, lr, r9, asr #31\n\t" - "aesd.8 q0, q3\n\t" - "aesimc.8 q0, q0\n\t" - "lsl r9, r9, #1\n\t" - "aesd.8 q0, q4\n\t" - "aesimc.8 q0, q0\n\t" - "orr r9, r9, r8, lsr #31\n\t" - "aesd.8 q0, q5\n\t" - "aesimc.8 q0, q0\n\t" - "lsl r8, r8, #1\n\t" - "aesd.8 q0, q6\n\t" - "aesimc.8 q0, q0\n\t" - "orr r8, r8, r7, lsr #31\n\t" - "aesd.8 q0, q7\n\t" - "aesimc.8 q0, q0\n\t" - "lsl r7, r7, #1\n\t" - "aesd.8 q0, q8\n\t" - "aesimc.8 q0, q0\n\t" - "orr r7, r7, r6, lsr #31\n\t" - "aesd.8 q0, q9\n\t" - "aesimc.8 q0, q0\n\t" - "eor r6, r5, r6, lsl #1\n\t" - "aesd.8 q0, q10\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q11\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q13}, [r12]!\n\t" - "aesd.8 q0, q12\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aesd.8 q0, q13\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q13}, [r12]!\n\t" - "aesd.8 q0, q14\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]\n\t" - "aesd.8 q0, q13\n\t" - "veor.32 q0, q0, q14\n\t" - "sub r12, r12, #48\n\t" - "veor.32 q0, q0, q1\n\t" - "vmov d2, r6, r7\n\t" - "vmov d3, r8, r9\n\t" - "subs r4, r4, #1\n\t" - "vst1.8 {q0}, [%[out]]!\n\t" - "bne L_aes_xts_decrypt_arm32_crypto_256_start_1_%=\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_256_done_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_xts_decrypt_arm32_crypto_256_partial_done_%=\n\t" - "and r5, lr, r9, asr #31\n\t" - "lsl r9, r9, #1\n\t" - "orr r9, r9, r8, lsr #31\n\t" - "lsl r8, r8, #1\n\t" - "orr r8, r8, r7, lsr #31\n\t" - "lsl r7, r7, #1\n\t" - "orr r7, r7, r6, lsr #31\n\t" - "eor r6, r5, r6, lsl #1\n\t" - "vmov d30, r6, r7\n\t" - "vmov d31, r8, r9\n\t" - "vld1.8 {q0}, [%[in]]!\n\t" - "ldr r4, [sp, #8]\n\t" - "veor.32 q0, q0, q15\n\t" - "aesd.8 q0, q2\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q3\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q4\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q5\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q6\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q7\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q8\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q9\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q10\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q11\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q13}, [r12]!\n\t" - "aesd.8 q0, q12\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aesd.8 q0, q13\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q13}, [r12]!\n\t" - "aesd.8 q0, q14\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]\n\t" - "aesd.8 q0, q13\n\t" - "veor.32 q0, q0, q14\n\t" - "sub r12, r12, #48\n\t" - "veor.32 q0, q0, q15\n\t" - "vst1.32 {q0}, [r4]\n\t" - "add %[out], %[out], #16\n\t" - "mov r5, %[sz]\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_256_start_byte_%=:\n\t" - "ldrb r8, [r4]\n\t" - "ldrb r9, [%[in]], #1\n\t" - "strb r8, [%[out]], #1\n\t" - "strb r9, [r4], #1\n\t" - "subs r5, r5, #1\n\t" - "bgt L_aes_xts_decrypt_arm32_crypto_256_start_byte_%=\n\t" - "sub %[out], %[out], %[sz]\n\t" - "sub r4, r4, %[sz]\n\t" - "sub %[out], %[out], #16\n\t" - "vld1.32 {q0}, [r4]\n\t" - "veor.32 q0, q0, q1\n\t" - "aesd.8 q0, q2\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q3\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q4\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q5\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q6\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q7\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q8\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q9\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q10\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q11\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q13}, [r12]!\n\t" - "aesd.8 q0, q12\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]!\n\t" - "aesd.8 q0, q13\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q13}, [r12]!\n\t" - "aesd.8 q0, q14\n\t" - "aesimc.8 q0, q0\n\t" - "vld1.32 {q14}, [r12]\n\t" - "aesd.8 q0, q13\n\t" - "veor.32 q0, q0, q14\n\t" - "veor.32 q0, q0, q1\n\t" - "vst1.8 {q0}, [%[out]]\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_256_partial_done_%=:\n\t" -#endif /* !NO_AES_256 */ - "b L_aes_xts_decrypt_arm32_crypto_done_%=\n\t" - /* AES_XTS_128 */ - "\n" - "L_aes_xts_decrypt_arm32_crypto_start_128_%=:\n\t" -#ifndef NO_AES_128 - "vldm.32 lr!, {q10-q12}\n\t" - "aese.8 q1, q2\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q3\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q4\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q6\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q7\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q8\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q9\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q10\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q1, q11\n\t" - "veor.32 q1, q1, q12\n\t" - "vmov r6, r7, d2\n\t" - "vmov r8, r9, d3\n\t" - "vldm.32 r12!, {q2-q9}\n\t" - "vldm.32 r12!, {q10-q12}\n\t" - "mov lr, #0x87\n\t" - "cmp r4, #1\n\t" - "blt L_aes_xts_decrypt_arm32_crypto_128_done_%=\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_128_start_1_%=:\n\t" - "vld1.8 {q0}, [%[in]]!\n\t" - "veor.32 q0, q0, q1\n\t" - "aesd.8 q0, q2\n\t" - "aesimc.8 q0, q0\n\t" - "and r5, lr, r9, asr #31\n\t" - "aesd.8 q0, q3\n\t" - "aesimc.8 q0, q0\n\t" - "lsl r9, r9, #1\n\t" - "aesd.8 q0, q4\n\t" - "aesimc.8 q0, q0\n\t" - "orr r9, r9, r8, lsr #31\n\t" - "aesd.8 q0, q5\n\t" - "aesimc.8 q0, q0\n\t" - "lsl r8, r8, #1\n\t" - "aesd.8 q0, q6\n\t" - "aesimc.8 q0, q0\n\t" - "orr r8, r8, r7, lsr #31\n\t" - "aesd.8 q0, q7\n\t" - "aesimc.8 q0, q0\n\t" - "lsl r7, r7, #1\n\t" - "aesd.8 q0, q8\n\t" - "aesimc.8 q0, q0\n\t" - "orr r7, r7, r6, lsr #31\n\t" - "aesd.8 q0, q9\n\t" - "aesimc.8 q0, q0\n\t" - "eor r6, r5, r6, lsl #1\n\t" - "aesd.8 q0, q10\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q11\n\t" - "veor.32 q0, q0, q12\n\t" - "veor.32 q0, q0, q1\n\t" - "vmov d2, r6, r7\n\t" - "vmov d3, r8, r9\n\t" - "subs r4, r4, #1\n\t" - "vst1.8 {q0}, [%[out]]!\n\t" - "bne L_aes_xts_decrypt_arm32_crypto_128_start_1_%=\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_128_done_%=:\n\t" - "cmp %[sz], #0\n\t" - "beq L_aes_xts_decrypt_arm32_crypto_128_partial_done_%=\n\t" - "and r5, lr, r9, asr #31\n\t" - "lsl r9, r9, #1\n\t" - "orr r9, r9, r8, lsr #31\n\t" - "lsl r8, r8, #1\n\t" - "orr r8, r8, r7, lsr #31\n\t" - "lsl r7, r7, #1\n\t" - "orr r7, r7, r6, lsr #31\n\t" - "eor r6, r5, r6, lsl #1\n\t" - "vmov d30, r6, r7\n\t" - "vmov d31, r8, r9\n\t" - "vld1.8 {q0}, [%[in]]!\n\t" - "ldr r4, [sp, #8]\n\t" - "veor.32 q0, q0, q15\n\t" - "aesd.8 q0, q2\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q3\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q4\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q5\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q6\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q7\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q8\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q9\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q10\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q11\n\t" - "veor.32 q0, q0, q12\n\t" - "veor.32 q0, q0, q15\n\t" - "vst1.32 {q0}, [r4]\n\t" - "add %[out], %[out], #16\n\t" - "mov r5, %[sz]\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_128_start_byte_%=:\n\t" - "ldrb r8, [r4]\n\t" - "ldrb r9, [%[in]], #1\n\t" - "strb r8, [%[out]], #1\n\t" - "strb r9, [r4], #1\n\t" - "subs r5, r5, #1\n\t" - "bgt L_aes_xts_decrypt_arm32_crypto_128_start_byte_%=\n\t" - "sub %[out], %[out], %[sz]\n\t" - "sub r4, r4, %[sz]\n\t" - "sub %[out], %[out], #16\n\t" - "vld1.32 {q0}, [r4]\n\t" - "veor.32 q0, q0, q1\n\t" - "aesd.8 q0, q2\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q3\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q4\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q5\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q6\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q7\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q8\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q9\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q10\n\t" - "aesimc.8 q0, q0\n\t" - "aesd.8 q0, q11\n\t" - "veor.32 q0, q0, q12\n\t" - "veor.32 q0, q0, q1\n\t" - "vst1.8 {q0}, [%[out]]\n\t" - "\n" - "L_aes_xts_decrypt_arm32_crypto_128_partial_done_%=:\n\t" -#endif /* !NO_AES_128 */ - "\n" - "L_aes_xts_decrypt_arm32_crypto_done_%=:\n\t" - "pop {%[key], %[key2]}\n\t" - "pop {%[tmp], %[nr]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [i] "+r" (i), - [key] "+r" (key), [key2] "+r" (key2), [tmp] "+r" (tmp), - [nr] "+r" (nr) - : -#else - : - : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [i] "r" (i), - [key] "r" (key), [key2] "r" (key2), [tmp] "r" (tmp), [nr] "r" (nr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r6", "r7", "r8", "r9", "q0", "q1", "q2", "q3", "q4", - "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", - "q15" - ); -} - -#endif /* HAVE_AES_DECRYPT */ -#endif /* WOLFSSL_AES_XTS */ -#ifdef WOLFSSL_AESGCM_SIV -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_GCMSIV_polyval_crypto(unsigned char* s_p, - const unsigned char* h_p, const unsigned char* data_p, - unsigned int blocks_p) -#else -WC_OMIT_FRAME_POINTER void AES_GCMSIV_polyval_crypto(unsigned char* s, - const unsigned char* h, const unsigned char* data, unsigned int blocks) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register unsigned char* s __asm__ ("r0") = (unsigned char*)s_p; - register const unsigned char* h __asm__ ("r1") = (const unsigned char*)h_p; - register const unsigned char* data __asm__ ("r2") = - (const unsigned char*)data_p; - register unsigned int blocks __asm__ ("r3") = (unsigned int)blocks_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "veor.8 q2, q2, q2\n\t" - "vld1.8 {q1}, [%[h]]\n\t" - "vld1.8 {q0}, [%[s]]\n\t" - "vrev64.8 q0, q0\n\t" - "vext.8 q0, q0, q0, #8\n\t" - "vmull.p64 q7, d2, d2\n\t" - "vmull.p64 q11, d2, d3\n\t" - "vmull.p64 q12, d3, d2\n\t" - "vmull.p64 q8, d3, d3\n\t" - "veor.8 q11, q11, q12\n\t" - "vext.8 q12, q2, q11, #8\n\t" - "vext.8 q11, q11, q2, #8\n\t" - "veor.8 q7, q7, q12\n\t" - "veor.8 q8, q8, q11\n\t" - "vshr.u32 q6, q7, #31\n\t" - "vshr.u32 q9, q8, #31\n\t" - "vshl.i32 q7, q7, #1\n\t" - "vshl.i32 q8, q8, #1\n\t" - "vext.8 q10, q6, q2, #12\n\t" - "vext.8 q9, q2, q9, #12\n\t" - "vext.8 q6, q2, q6, #12\n\t" - "veor.8 q7, q7, q6\n\t" - "veor.8 q8, q8, q9\n\t" - "veor.8 q8, q8, q10\n\t" - "vshl.i32 q6, q7, #31\n\t" - "vshl.i32 q9, q7, #30\n\t" - "vshl.i32 q10, q7, #25\n\t" - "veor.8 q6, q6, q9\n\t" - "veor.8 q6, q6, q10\n\t" - "vext.8 q9, q6, q2, #4\n\t" - "vext.8 q6, q2, q6, #4\n\t" - "veor.8 q7, q7, q6\n\t" - "vshr.u32 q11, q7, #1\n\t" - "vshr.u32 q12, q7, #2\n\t" - "vshr.u32 q13, q7, #7\n\t" - "veor.8 q11, q11, q12\n\t" - "veor.8 q11, q11, q13\n\t" - "veor.8 q11, q11, q9\n\t" - "veor.8 q7, q7, q11\n\t" - "veor.8 q3, q8, q7\n\t" - "vmull.p64 q7, d6, d2\n\t" - "vmull.p64 q11, d6, d3\n\t" - "vmull.p64 q12, d7, d2\n\t" - "vmull.p64 q8, d7, d3\n\t" - "veor.8 q11, q11, q12\n\t" - "vext.8 q12, q2, q11, #8\n\t" - "vext.8 q11, q11, q2, #8\n\t" - "veor.8 q7, q7, q12\n\t" - "veor.8 q8, q8, q11\n\t" - "vshr.u32 q6, q7, #31\n\t" - "vshr.u32 q9, q8, #31\n\t" - "vshl.i32 q7, q7, #1\n\t" - "vshl.i32 q8, q8, #1\n\t" - "vext.8 q10, q6, q2, #12\n\t" - "vext.8 q9, q2, q9, #12\n\t" - "vext.8 q6, q2, q6, #12\n\t" - "veor.8 q7, q7, q6\n\t" - "veor.8 q8, q8, q9\n\t" - "veor.8 q8, q8, q10\n\t" - "vshl.i32 q6, q7, #31\n\t" - "vshl.i32 q9, q7, #30\n\t" - "vshl.i32 q10, q7, #25\n\t" - "veor.8 q6, q6, q9\n\t" - "veor.8 q6, q6, q10\n\t" - "vext.8 q9, q6, q2, #4\n\t" - "vext.8 q6, q2, q6, #4\n\t" - "veor.8 q7, q7, q6\n\t" - "vshr.u32 q11, q7, #1\n\t" - "vshr.u32 q12, q7, #2\n\t" - "vshr.u32 q13, q7, #7\n\t" - "veor.8 q11, q11, q12\n\t" - "veor.8 q11, q11, q13\n\t" - "veor.8 q11, q11, q9\n\t" - "veor.8 q7, q7, q11\n\t" - "veor.8 q4, q8, q7\n\t" - "vmull.p64 q7, d6, d6\n\t" - "vmull.p64 q11, d6, d7\n\t" - "vmull.p64 q12, d7, d6\n\t" - "vmull.p64 q8, d7, d7\n\t" - "veor.8 q11, q11, q12\n\t" - "vext.8 q12, q2, q11, #8\n\t" - "vext.8 q11, q11, q2, #8\n\t" - "veor.8 q7, q7, q12\n\t" - "veor.8 q8, q8, q11\n\t" - "vshr.u32 q6, q7, #31\n\t" - "vshr.u32 q9, q8, #31\n\t" - "vshl.i32 q7, q7, #1\n\t" - "vshl.i32 q8, q8, #1\n\t" - "vext.8 q10, q6, q2, #12\n\t" - "vext.8 q9, q2, q9, #12\n\t" - "vext.8 q6, q2, q6, #12\n\t" - "veor.8 q7, q7, q6\n\t" - "veor.8 q8, q8, q9\n\t" - "veor.8 q8, q8, q10\n\t" - "vshl.i32 q6, q7, #31\n\t" - "vshl.i32 q9, q7, #30\n\t" - "vshl.i32 q10, q7, #25\n\t" - "veor.8 q6, q6, q9\n\t" - "veor.8 q6, q6, q10\n\t" - "vext.8 q9, q6, q2, #4\n\t" - "vext.8 q6, q2, q6, #4\n\t" - "veor.8 q7, q7, q6\n\t" - "vshr.u32 q11, q7, #1\n\t" - "vshr.u32 q12, q7, #2\n\t" - "vshr.u32 q13, q7, #7\n\t" - "veor.8 q11, q11, q12\n\t" - "veor.8 q11, q11, q13\n\t" - "veor.8 q11, q11, q9\n\t" - "veor.8 q7, q7, q11\n\t" - "veor.8 q5, q8, q7\n\t" - "lsr r12, %[blocks], #2\n\t" - "cmp r12, #0\n\t" - "beq L_AES_GCMSIV_polyval_crypto_rem_start_%=\n\t" - "\n" - "L_AES_GCMSIV_polyval_crypto_group_%=:\n\t" - "vld1.8 {q6}, [%[data]]!\n\t" - "veor.8 q6, q6, q0\n\t" - "vmull.p64 q9, d12, d10\n\t" - "vmull.p64 q11, d12, d11\n\t" - "vmull.p64 q12, d13, d10\n\t" - "vmull.p64 q10, d13, d11\n\t" - "veor.8 q11, q11, q12\n\t" - "vext.8 q12, q2, q11, #8\n\t" - "vext.8 q11, q11, q2, #8\n\t" - "veor.8 q9, q9, q12\n\t" - "veor.8 q10, q10, q11\n\t" - "vld1.8 {q6}, [%[data]]!\n\t" - "vmull.p64 q7, d12, d8\n\t" - "vmull.p64 q11, d12, d9\n\t" - "vmull.p64 q12, d13, d8\n\t" - "vmull.p64 q8, d13, d9\n\t" - "veor.8 q11, q11, q12\n\t" - "vext.8 q12, q2, q11, #8\n\t" - "vext.8 q11, q11, q2, #8\n\t" - "veor.8 q7, q7, q12\n\t" - "veor.8 q8, q8, q11\n\t" - "veor.8 q9, q9, q7\n\t" - "veor.8 q10, q10, q8\n\t" - "vld1.8 {q6}, [%[data]]!\n\t" - "vmull.p64 q7, d12, d6\n\t" - "vmull.p64 q11, d12, d7\n\t" - "vmull.p64 q12, d13, d6\n\t" - "vmull.p64 q8, d13, d7\n\t" - "veor.8 q11, q11, q12\n\t" - "vext.8 q12, q2, q11, #8\n\t" - "vext.8 q11, q11, q2, #8\n\t" - "veor.8 q7, q7, q12\n\t" - "veor.8 q8, q8, q11\n\t" - "veor.8 q9, q9, q7\n\t" - "veor.8 q10, q10, q8\n\t" - "vld1.8 {q6}, [%[data]]!\n\t" - "vmull.p64 q7, d12, d2\n\t" - "vmull.p64 q11, d12, d3\n\t" - "vmull.p64 q12, d13, d2\n\t" - "vmull.p64 q8, d13, d3\n\t" - "veor.8 q11, q11, q12\n\t" - "vext.8 q12, q2, q11, #8\n\t" - "vext.8 q11, q11, q2, #8\n\t" - "veor.8 q7, q7, q12\n\t" - "veor.8 q8, q8, q11\n\t" - "veor.8 q9, q9, q7\n\t" - "veor.8 q10, q10, q8\n\t" - "vshr.u32 q6, q9, #31\n\t" - "vshr.u32 q7, q10, #31\n\t" - "vshl.i32 q9, q9, #1\n\t" - "vshl.i32 q10, q10, #1\n\t" - "vext.8 q8, q6, q2, #12\n\t" - "vext.8 q7, q2, q7, #12\n\t" - "vext.8 q6, q2, q6, #12\n\t" - "veor.8 q9, q9, q6\n\t" - "veor.8 q10, q10, q7\n\t" - "veor.8 q10, q10, q8\n\t" - "vshl.i32 q6, q9, #31\n\t" - "vshl.i32 q7, q9, #30\n\t" - "vshl.i32 q8, q9, #25\n\t" - "veor.8 q6, q6, q7\n\t" - "veor.8 q6, q6, q8\n\t" - "vext.8 q7, q6, q2, #4\n\t" - "vext.8 q6, q2, q6, #4\n\t" - "veor.8 q9, q9, q6\n\t" - "vshr.u32 q11, q9, #1\n\t" - "vshr.u32 q12, q9, #2\n\t" - "vshr.u32 q13, q9, #7\n\t" - "veor.8 q11, q11, q12\n\t" - "veor.8 q11, q11, q13\n\t" - "veor.8 q11, q11, q7\n\t" - "veor.8 q9, q9, q11\n\t" - "veor.8 q0, q10, q9\n\t" - "subs r12, r12, #1\n\t" - "bne L_AES_GCMSIV_polyval_crypto_group_%=\n\t" - "\n" - "L_AES_GCMSIV_polyval_crypto_rem_start_%=:\n\t" - "and %[blocks], %[blocks], #3\n\t" - "cmp %[blocks], #0\n\t" - "beq L_AES_GCMSIV_polyval_crypto_done_%=\n\t" - "\n" - "L_AES_GCMSIV_polyval_crypto_rem_%=:\n\t" - "vld1.8 {q6}, [%[data]]!\n\t" - "veor.8 q0, q0, q6\n\t" - "vmull.p64 q9, d0, d2\n\t" - "vmull.p64 q11, d0, d3\n\t" - "vmull.p64 q12, d1, d2\n\t" - "vmull.p64 q10, d1, d3\n\t" - "veor.8 q11, q11, q12\n\t" - "vext.8 q12, q2, q11, #8\n\t" - "vext.8 q11, q11, q2, #8\n\t" - "veor.8 q9, q9, q12\n\t" - "veor.8 q10, q10, q11\n\t" - "vshr.u32 q6, q9, #31\n\t" - "vshr.u32 q7, q10, #31\n\t" - "vshl.i32 q9, q9, #1\n\t" - "vshl.i32 q10, q10, #1\n\t" - "vext.8 q8, q6, q2, #12\n\t" - "vext.8 q7, q2, q7, #12\n\t" - "vext.8 q6, q2, q6, #12\n\t" - "veor.8 q9, q9, q6\n\t" - "veor.8 q10, q10, q7\n\t" - "veor.8 q10, q10, q8\n\t" - "vshl.i32 q6, q9, #31\n\t" - "vshl.i32 q7, q9, #30\n\t" - "vshl.i32 q8, q9, #25\n\t" - "veor.8 q6, q6, q7\n\t" - "veor.8 q6, q6, q8\n\t" - "vext.8 q7, q6, q2, #4\n\t" - "vext.8 q6, q2, q6, #4\n\t" - "veor.8 q9, q9, q6\n\t" - "vshr.u32 q11, q9, #1\n\t" - "vshr.u32 q12, q9, #2\n\t" - "vshr.u32 q13, q9, #7\n\t" - "veor.8 q11, q11, q12\n\t" - "veor.8 q11, q11, q13\n\t" - "veor.8 q11, q11, q7\n\t" - "veor.8 q9, q9, q11\n\t" - "veor.8 q0, q10, q9\n\t" - "subs %[blocks], %[blocks], #1\n\t" - "bne L_AES_GCMSIV_polyval_crypto_rem_%=\n\t" - "\n" - "L_AES_GCMSIV_polyval_crypto_done_%=:\n\t" - "vrev64.8 q0, q0\n\t" - "vext.8 q0, q0, q0, #8\n\t" - "vst1.8 {q0}, [%[s]]\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [s] "+r" (s), [h] "+r" (h), [data] "+r" (data), - [blocks] "+r" (blocks) - : -#else - : - : [s] "r" (s), [h] "r" (h), [data] "r" (data), [blocks] "r" (blocks) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r12", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", - "q8", "q9", "q10", "q11", "q12", "q13" - ); -} - -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_crypto(const unsigned char* in_p, - unsigned char* out_p, unsigned long length_p, const unsigned char* KS_p, - int nr_p, unsigned char* ctr_p) -#else -WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_crypto(const unsigned char* in, - unsigned char* out, unsigned long length, const unsigned char* KS, int nr, - unsigned char* ctr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const unsigned char* in __asm__ ("r0") = - (const unsigned char*)in_p; - register unsigned char* out __asm__ ("r1") = (unsigned char*)out_p; - register unsigned long length __asm__ ("r2") = (unsigned long)length_p; - register const unsigned char* KS __asm__ ("r3") = - (const unsigned char*)KS_p; - register int nr __asm__ ("r12") = (int)nr_p; - register unsigned char* ctr __asm__ ("lr") = (unsigned char*)ctr_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[nr], %[ctr]}\n\t" - "ldr r12, [sp]\n\t" - "ldr lr, [sp, #4]\n\t" - "vld1.8 {q0}, [lr]\n\t" - "lsr r5, %[length], #4\n\t" - "vmov r7, r8, d0\n\t" - "vmov r9, r10, d1\n\t" - "\n" - "L_AES_GCMSIV_ctr_crypto_loop2_%=:\n\t" - "cmp r5, #2\n\t" - "blt L_AES_GCMSIV_ctr_crypto_tail_%=\n\t" - "vmov d2, r7, r8\n\t" - "vmov d3, r9, r10\n\t" - "add %[length], r7, #1\n\t" - "vmov d4, %[length], r8\n\t" - "vmov d5, r9, r10\n\t" - "add r7, r7, #2\n\t" - "mov r4, %[KS]\n\t" - "sub r6, r12, #1\n\t" - "\n" - "L_AES_GCMSIV_ctr_crypto_rounds2_%=:\n\t" - "vld1.32 {q5}, [r4]!\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "aese.8 q2, q5\n\t" - "aesmc.8 q2, q2\n\t" - "subs r6, r6, #1\n\t" - "bne L_AES_GCMSIV_ctr_crypto_rounds2_%=\n\t" - "vld1.32 {q5}, [r4]!\n\t" - "aese.8 q1, q5\n\t" - "aese.8 q2, q5\n\t" - "vld1.32 {q5}, [r4]\n\t" - "veor.8 q1, q1, q5\n\t" - "veor.8 q2, q2, q5\n\t" - "vld1.8 {q3-q4}, [%[in]]!\n\t" - "veor.8 q3, q3, q1\n\t" - "veor.8 q4, q4, q2\n\t" - "vst1.8 {q3-q4}, [%[out]]!\n\t" - "sub r5, r5, #2\n\t" - "b L_AES_GCMSIV_ctr_crypto_loop2_%=\n\t" - "\n" - "L_AES_GCMSIV_ctr_crypto_tail_%=:\n\t" - "cmp r5, #0\n\t" - "beq L_AES_GCMSIV_ctr_crypto_done_%=\n\t" - "vmov d2, r7, r8\n\t" - "vmov d3, r9, r10\n\t" - "add r7, r7, #1\n\t" - "mov r4, %[KS]\n\t" - "sub r6, r12, #1\n\t" - "\n" - "L_AES_GCMSIV_ctr_crypto_rounds1_%=:\n\t" - "vld1.32 {q5}, [r4]!\n\t" - "aese.8 q1, q5\n\t" - "aesmc.8 q1, q1\n\t" - "subs r6, r6, #1\n\t" - "bne L_AES_GCMSIV_ctr_crypto_rounds1_%=\n\t" - "vld1.32 {q5}, [r4]!\n\t" - "aese.8 q1, q5\n\t" - "vld1.32 {q5}, [r4]\n\t" - "veor.8 q1, q1, q5\n\t" - "vld1.8 {q3}, [%[in]]!\n\t" - "veor.8 q3, q3, q1\n\t" - "vst1.8 {q3}, [%[out]]!\n\t" - "\n" - "L_AES_GCMSIV_ctr_crypto_done_%=:\n\t" - "vmov d0, r7, r8\n\t" - "vst1.8 {q0}, [lr]\n\t" - "pop {%[nr], %[ctr]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [length] "+r" (length), - [KS] "+r" (KS), [nr] "+r" (nr), [ctr] "+r" (ctr) - : -#else - : - : [in] "r" (in), [out] "r" (out), [length] "r" (length), [KS] "r" (KS), - [nr] "r" (nr), [ctr] "r" (ctr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "q0", "q1", - "q2", "q3", "q4", "q5" - ); -} - -#endif /* WOLFSSL_AESGCM_SIV */ -#else -#ifdef HAVE_AES_DECRYPT -XALIGNED(8) static const word32 L_AES_ARM32_td_data[] = { - 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e, - 0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303, - 0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c, - 0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3, - 0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0, - 0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9, - 0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259, - 0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8, - 0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971, - 0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a, - 0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f, - 0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b, - 0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8, - 0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab, - 0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708, - 0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682, - 0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2, - 0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe, - 0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb, - 0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10, - 0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd, - 0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015, - 0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e, - 0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee, - 0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000, - 0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72, - 0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39, - 0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e, - 0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91, - 0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a, - 0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17, - 0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9, - 0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60, - 0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e, - 0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1, - 0xcad731dc, 0x10426385, 0x40139722, 0x2084c611, - 0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1, - 0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3, - 0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964, - 0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390, - 0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b, - 0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf, - 0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46, - 0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af, - 0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512, - 0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb, - 0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a, - 0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8, - 0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c, - 0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266, - 0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8, - 0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6, - 0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604, - 0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551, - 0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41, - 0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647, - 0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c, - 0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1, - 0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737, - 0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db, - 0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340, - 0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95, - 0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1, - 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857, -}; - -#endif /* HAVE_AES_DECRYPT */ -#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AES_CBC) || \ - defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ - defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) -XALIGNED(8) static const word32 L_AES_ARM32_te_data[] = { - 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, - 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, - 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, - 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, - 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, - 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, - 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, - 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, - 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, - 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, - 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, - 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, - 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, - 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, - 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, - 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, - 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, - 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, - 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, - 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, - 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, - 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, - 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, - 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, - 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, - 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, - 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, - 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, - 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, - 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, - 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, - 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, - 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, - 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, - 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, - 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, - 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, - 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, - 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, - 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, - 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, - 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, - 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, - 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, - 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, - 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, - 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, - 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, - 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, - 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, - 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, - 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, - 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, - 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, - 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, - 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, - 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, - 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, - 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, - 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, - 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, - 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, - 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, - 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, -}; - -#endif /* HAVE_AES_DECRYPT || HAVE_AES_CBC || HAVE_AESCCM || HAVE_AESGCM || - * WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */ -#ifdef HAVE_AES_DECRYPT -static const word32* L_AES_ARM32_td = L_AES_ARM32_td_data; -#endif /* HAVE_AES_DECRYPT */ -#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AES_CBC) || \ - defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ - defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) -static const word32* L_AES_ARM32_te = L_AES_ARM32_te_data; -#endif /* HAVE_AES_DECRYPT || HAVE_AES_CBC || HAVE_AESCCM || HAVE_AESGCM || - * WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */ -#ifdef HAVE_AES_DECRYPT -void AES_invert_key(unsigned char* ks_p, word32 rounds_p); -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_invert_key(unsigned char* ks_p, word32 rounds_p) -#else -WC_OMIT_FRAME_POINTER void AES_invert_key(unsigned char* ks, word32 rounds) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register unsigned char* ks __asm__ ("r0") = (unsigned char*)ks_p; - register word32 rounds __asm__ ("r1") = (word32)rounds_p; - register word32* L_AES_ARM32_te_c __asm__ ("r2") = (word32*)L_AES_ARM32_te; - register word32* L_AES_ARM32_td_c __asm__ ("r3") = (word32*)L_AES_ARM32_td; -#else - register word32* L_AES_ARM32_te_c = (word32*)L_AES_ARM32_te; - register word32* L_AES_ARM32_td_c = (word32*)L_AES_ARM32_td; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "mov r12, %[L_AES_ARM32_te]\n\t" - "mov lr, %[L_AES_ARM32_td]\n\t" - "add r10, %[ks], %[rounds], lsl #4\n\t" - "mov r11, %[rounds]\n\t" - "\n" - "L_AES_invert_key_loop_%=:\n\t" - "ldm %[ks], {r2, r3, r4, r5}\n\t" - "ldm r10, {r6, r7, r8, r9}\n\t" - "stm r10, {r2, r3, r4, r5}\n\t" - "stm %[ks]!, {r6, r7, r8, r9}\n\t" - "subs r11, r11, #2\n\t" - "sub r10, r10, #16\n\t" - "bne L_AES_invert_key_loop_%=\n\t" - "sub %[ks], %[ks], %[rounds], lsl #3\n\t" - "add %[ks], %[ks], #16\n\t" - "sub r11, %[rounds], #1\n\t" - "\n" - "L_AES_invert_key_mix_loop_%=:\n\t" - "ldm %[ks], {r2, r3, r4, r5}\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r2, #24\n\t" - "lsr r6, r6, #24\n\t" -#else - "uxtb r6, r2\n\t" -#endif -#else - "ubfx r6, r2, #0, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r2, #16\n\t" - "lsr r7, r7, #24\n\t" -#else - "uxtb r7, r2, ror #8\n\t" -#endif -#else - "ubfx r7, r2, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r8, r2, #8\n\t" - "lsr r8, r8, #24\n\t" -#else - "uxtb r8, r2, ror #16\n\t" -#endif -#else - "ubfx r8, r2, #16, #8\n\t" -#endif - "lsr r9, r2, #24\n\t" - "ldrb r6, [r12, r6, lsl #2]\n\t" - "ldrb r7, [r12, r7, lsl #2]\n\t" - "ldrb r8, [r12, r8, lsl #2]\n\t" - "ldrb r9, [r12, r9, lsl #2]\n\t" - "ldr r6, [lr, r6, lsl #2]\n\t" - "ldr r7, [lr, r7, lsl #2]\n\t" - "ldr r8, [lr, r8, lsl #2]\n\t" - "ldr r9, [lr, r9, lsl #2]\n\t" - "eor r8, r8, r6, ror #16\n\t" - "eor r8, r8, r7, ror #8\n\t" - "eor r8, r8, r9, ror #24\n\t" - "str r8, [%[ks]], #4\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r3, #24\n\t" - "lsr r6, r6, #24\n\t" -#else - "uxtb r6, r3\n\t" -#endif -#else - "ubfx r6, r3, #0, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r3, #16\n\t" - "lsr r7, r7, #24\n\t" -#else - "uxtb r7, r3, ror #8\n\t" -#endif -#else - "ubfx r7, r3, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r8, r3, #8\n\t" - "lsr r8, r8, #24\n\t" -#else - "uxtb r8, r3, ror #16\n\t" -#endif -#else - "ubfx r8, r3, #16, #8\n\t" -#endif - "lsr r9, r3, #24\n\t" - "ldrb r6, [r12, r6, lsl #2]\n\t" - "ldrb r7, [r12, r7, lsl #2]\n\t" - "ldrb r8, [r12, r8, lsl #2]\n\t" - "ldrb r9, [r12, r9, lsl #2]\n\t" - "ldr r6, [lr, r6, lsl #2]\n\t" - "ldr r7, [lr, r7, lsl #2]\n\t" - "ldr r8, [lr, r8, lsl #2]\n\t" - "ldr r9, [lr, r9, lsl #2]\n\t" - "eor r8, r8, r6, ror #16\n\t" - "eor r8, r8, r7, ror #8\n\t" - "eor r8, r8, r9, ror #24\n\t" - "str r8, [%[ks]], #4\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r4, #24\n\t" - "lsr r6, r6, #24\n\t" -#else - "uxtb r6, r4\n\t" -#endif -#else - "ubfx r6, r4, #0, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r4, #16\n\t" - "lsr r7, r7, #24\n\t" -#else - "uxtb r7, r4, ror #8\n\t" -#endif -#else - "ubfx r7, r4, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r8, r4, #8\n\t" - "lsr r8, r8, #24\n\t" -#else - "uxtb r8, r4, ror #16\n\t" -#endif -#else - "ubfx r8, r4, #16, #8\n\t" -#endif - "lsr r9, r4, #24\n\t" - "ldrb r6, [r12, r6, lsl #2]\n\t" - "ldrb r7, [r12, r7, lsl #2]\n\t" - "ldrb r8, [r12, r8, lsl #2]\n\t" - "ldrb r9, [r12, r9, lsl #2]\n\t" - "ldr r6, [lr, r6, lsl #2]\n\t" - "ldr r7, [lr, r7, lsl #2]\n\t" - "ldr r8, [lr, r8, lsl #2]\n\t" - "ldr r9, [lr, r9, lsl #2]\n\t" - "eor r8, r8, r6, ror #16\n\t" - "eor r8, r8, r7, ror #8\n\t" - "eor r8, r8, r9, ror #24\n\t" - "str r8, [%[ks]], #4\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r5, #24\n\t" - "lsr r6, r6, #24\n\t" -#else - "uxtb r6, r5\n\t" -#endif -#else - "ubfx r6, r5, #0, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r5, #16\n\t" - "lsr r7, r7, #24\n\t" -#else - "uxtb r7, r5, ror #8\n\t" -#endif -#else - "ubfx r7, r5, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r8, r5, #8\n\t" - "lsr r8, r8, #24\n\t" -#else - "uxtb r8, r5, ror #16\n\t" -#endif -#else - "ubfx r8, r5, #16, #8\n\t" -#endif - "lsr r9, r5, #24\n\t" - "ldrb r6, [r12, r6, lsl #2]\n\t" - "ldrb r7, [r12, r7, lsl #2]\n\t" - "ldrb r8, [r12, r8, lsl #2]\n\t" - "ldrb r9, [r12, r9, lsl #2]\n\t" - "ldr r6, [lr, r6, lsl #2]\n\t" - "ldr r7, [lr, r7, lsl #2]\n\t" - "ldr r8, [lr, r8, lsl #2]\n\t" - "ldr r9, [lr, r9, lsl #2]\n\t" - "eor r8, r8, r6, ror #16\n\t" - "eor r8, r8, r7, ror #8\n\t" - "eor r8, r8, r9, ror #24\n\t" - "str r8, [%[ks]], #4\n\t" - "subs r11, r11, #1\n\t" - "bne L_AES_invert_key_mix_loop_%=\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [ks] "+r" (ks), [rounds] "+r" (rounds), - [L_AES_ARM32_te] "+r" (L_AES_ARM32_te_c), - [L_AES_ARM32_td] "+r" (L_AES_ARM32_td_c) - : -#else - : - : [ks] "r" (ks), [rounds] "r" (rounds), - [L_AES_ARM32_te] "r" (L_AES_ARM32_te_c), - [L_AES_ARM32_td] "r" (L_AES_ARM32_td_c) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r12", "lr", "r4", "r5", "r6", "r7", "r8", "r9", - "r10", "r11" - ); -} - -#endif /* HAVE_AES_DECRYPT */ -XALIGNED(8) static const word32 L_AES_ARM32_rcon[] = { - 0x01000000, 0x02000000, 0x04000000, 0x08000000, - 0x10000000, 0x20000000, 0x40000000, 0x80000000, - 0x1b000000, 0x36000000 -}; - -void AES_set_encrypt_key(const unsigned char* key_p, word32 len_p, - unsigned char* ks_p); -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_set_encrypt_key(const unsigned char* key_p, - word32 len_p, unsigned char* ks_p) -#else -WC_OMIT_FRAME_POINTER void AES_set_encrypt_key(const unsigned char* key, - word32 len, unsigned char* ks) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const unsigned char* key __asm__ ("r0") = - (const unsigned char*)key_p; - register word32 len __asm__ ("r1") = (word32)len_p; - register unsigned char* ks __asm__ ("r2") = (unsigned char*)ks_p; - register word32* L_AES_ARM32_te_c __asm__ ("r3") = (word32*)L_AES_ARM32_te; - register word32* L_AES_ARM32_rcon_c __asm__ ("r12") = - (word32*)&L_AES_ARM32_rcon; -#else - register word32* L_AES_ARM32_te_c = (word32*)L_AES_ARM32_te; - register word32* L_AES_ARM32_rcon_c = (word32*)&L_AES_ARM32_rcon; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[L_AES_ARM32_rcon]}\n\t" - "mov r8, %[L_AES_ARM32_te]\n\t" - "mov lr, %[L_AES_ARM32_rcon]\n\t" - "cmp %[len], #0x80\n\t" - "beq L_AES_set_encrypt_key_start_128_%=\n\t" - "cmp %[len], #0xc0\n\t" - "beq L_AES_set_encrypt_key_start_192_%=\n\t" - "ldr r4, [%[key]]\n\t" - "ldr r5, [%[key], #4]\n\t" - "ldr r6, [%[key], #8]\n\t" - "ldr r7, [%[key], #12]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - /* REV r4, r4 */ - "eor r3, r4, r4, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "eor r4, r4, r3, lsr #8\n\t" - /* REV r5, r5 */ - "eor r3, r5, r5, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r5, r5, #8\n\t" - "eor r5, r5, r3, lsr #8\n\t" - /* REV r6, r6 */ - "eor r3, r6, r6, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r6, r6, #8\n\t" - "eor r6, r6, r3, lsr #8\n\t" - /* REV r7, r7 */ - "eor r3, r7, r7, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r7, r7, #8\n\t" - "eor r7, r7, r3, lsr #8\n\t" -#else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "stm %[ks]!, {r4, r5, r6, r7}\n\t" - "ldr r4, [%[key], #16]\n\t" - "ldr r5, [%[key], #20]\n\t" - "ldr r6, [%[key], #24]\n\t" - "ldr r7, [%[key], #28]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - /* REV r4, r4 */ - "eor r3, r4, r4, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "eor r4, r4, r3, lsr #8\n\t" - /* REV r5, r5 */ - "eor r3, r5, r5, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r5, r5, #8\n\t" - "eor r5, r5, r3, lsr #8\n\t" - /* REV r6, r6 */ - "eor r3, r6, r6, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r6, r6, #8\n\t" - "eor r6, r6, r3, lsr #8\n\t" - /* REV r7, r7 */ - "eor r3, r7, r7, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r7, r7, #8\n\t" - "eor r7, r7, r3, lsr #8\n\t" -#else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "stm %[ks], {r4, r5, r6, r7}\n\t" - "sub %[ks], %[ks], #16\n\t" - "mov r12, #6\n\t" - "\n" - "L_AES_set_encrypt_key_loop_256_%=:\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r7, #24\n\t" - "lsr r4, r4, #24\n\t" -#else - "uxtb r4, r7\n\t" -#endif -#else - "ubfx r4, r7, #0, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r7, #16\n\t" - "lsr r5, r5, #24\n\t" -#else - "uxtb r5, r7, ror #8\n\t" -#endif -#else - "ubfx r5, r7, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r7, #8\n\t" - "lsr r6, r6, #24\n\t" -#else - "uxtb r6, r7, ror #16\n\t" -#endif -#else - "ubfx r6, r7, #16, #8\n\t" -#endif - "lsr r7, r7, #24\n\t" - "ldrb r4, [r8, r4, lsl #2]\n\t" - "ldrb r5, [r8, r5, lsl #2]\n\t" - "ldrb r6, [r8, r6, lsl #2]\n\t" - "ldrb r7, [r8, r7, lsl #2]\n\t" - "eor r3, r7, r4, lsl #8\n\t" - "eor r3, r3, r5, lsl #16\n\t" - "eor r3, r3, r6, lsl #24\n\t" - "ldm %[ks]!, {r4, r5, r6, r7}\n\t" - "eor r4, r4, r3\n\t" - "ldm lr!, {r3}\n\t" - "eor r4, r4, r3\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "add %[ks], %[ks], #16\n\t" - "stm %[ks], {r4, r5, r6, r7}\n\t" - "sub %[ks], %[ks], #16\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r7, #16\n\t" - "lsr r4, r4, #24\n\t" -#else - "uxtb r4, r7, ror #8\n\t" -#endif -#else - "ubfx r4, r7, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r7, #8\n\t" - "lsr r5, r5, #24\n\t" -#else - "uxtb r5, r7, ror #16\n\t" -#endif -#else - "ubfx r5, r7, #16, #8\n\t" -#endif - "lsr r6, r7, #24\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r3, r7, #24\n\t" - "lsr r3, r3, #24\n\t" -#else - "uxtb r3, r7\n\t" -#endif -#else - "ubfx r3, r7, #0, #8\n\t" -#endif - "ldrb r4, [r8, r4, lsl #2]\n\t" - "ldrb r6, [r8, r6, lsl #2]\n\t" - "ldrb r5, [r8, r5, lsl #2]\n\t" - "ldrb r3, [r8, r3, lsl #2]\n\t" - "eor r3, r3, r4, lsl #8\n\t" - "eor r3, r3, r5, lsl #16\n\t" - "eor r3, r3, r6, lsl #24\n\t" - "ldm %[ks]!, {r4, r5, r6, r7}\n\t" - "eor r4, r4, r3\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "add %[ks], %[ks], #16\n\t" - "stm %[ks], {r4, r5, r6, r7}\n\t" - "sub %[ks], %[ks], #16\n\t" - "subs r12, r12, #1\n\t" - "bne L_AES_set_encrypt_key_loop_256_%=\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r7, #24\n\t" - "lsr r4, r4, #24\n\t" -#else - "uxtb r4, r7\n\t" -#endif -#else - "ubfx r4, r7, #0, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r7, #16\n\t" - "lsr r5, r5, #24\n\t" -#else - "uxtb r5, r7, ror #8\n\t" -#endif -#else - "ubfx r5, r7, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r7, #8\n\t" - "lsr r6, r6, #24\n\t" -#else - "uxtb r6, r7, ror #16\n\t" -#endif -#else - "ubfx r6, r7, #16, #8\n\t" -#endif - "lsr r7, r7, #24\n\t" - "ldrb r4, [r8, r4, lsl #2]\n\t" - "ldrb r5, [r8, r5, lsl #2]\n\t" - "ldrb r6, [r8, r6, lsl #2]\n\t" - "ldrb r7, [r8, r7, lsl #2]\n\t" - "eor r3, r7, r4, lsl #8\n\t" - "eor r3, r3, r5, lsl #16\n\t" - "eor r3, r3, r6, lsl #24\n\t" - "ldm %[ks]!, {r4, r5, r6, r7}\n\t" - "eor r4, r4, r3\n\t" - "ldm lr!, {r3}\n\t" - "eor r4, r4, r3\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "add %[ks], %[ks], #16\n\t" - "stm %[ks], {r4, r5, r6, r7}\n\t" - "sub %[ks], %[ks], #16\n\t" - "b L_AES_set_encrypt_key_end_%=\n\t" - "\n" - "L_AES_set_encrypt_key_start_192_%=:\n\t" - "ldr r4, [%[key]]\n\t" - "ldr r5, [%[key], #4]\n\t" - "ldr r6, [%[key], #8]\n\t" - "ldr r7, [%[key], #12]\n\t" - "ldr %[len], [%[key], #20]\n\t" - "ldr %[key], [%[key], #16]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - /* REV r4, r4 */ - "eor r3, r4, r4, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "eor r4, r4, r3, lsr #8\n\t" - /* REV r5, r5 */ - "eor r3, r5, r5, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r5, r5, #8\n\t" - "eor r5, r5, r3, lsr #8\n\t" - /* REV r6, r6 */ - "eor r3, r6, r6, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r6, r6, #8\n\t" - "eor r6, r6, r3, lsr #8\n\t" - /* REV r7, r7 */ - "eor r3, r7, r7, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r7, r7, #8\n\t" - "eor r7, r7, r3, lsr #8\n\t" - /* REV r0, r0 */ - "eor r3, %[key], %[key], ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror %[key], %[key], #8\n\t" - "eor %[key], %[key], r3, lsr #8\n\t" - /* REV r1, r1 */ - "eor r3, %[len], %[len], ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror %[len], %[len], #8\n\t" - "eor %[len], %[len], r3, lsr #8\n\t" -#else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" - "rev %[key], %[key]\n\t" - "rev %[len], %[len]\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "stm %[ks], {r4, r5, r6, r7}\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "str %[key], [%[ks], #16]\n\t" - "str %[len], [%[ks], #20]\n\t" -#else - "strd %[key], %[len], [%[ks], #16]\n\t" -#endif - "mov r7, %[len]\n\t" - "mov r12, #7\n\t" - "\n" - "L_AES_set_encrypt_key_loop_192_%=:\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r0, r7, #24\n\t" - "lsr r0, r0, #24\n\t" -#else - "uxtb r0, r7\n\t" -#endif -#else - "ubfx r0, r7, #0, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r1, r7, #16\n\t" - "lsr r1, r1, #24\n\t" -#else - "uxtb r1, r7, ror #8\n\t" -#endif -#else - "ubfx r1, r7, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r7, #8\n\t" - "lsr r4, r4, #24\n\t" -#else - "uxtb r4, r7, ror #16\n\t" -#endif -#else - "ubfx r4, r7, #16, #8\n\t" -#endif - "lsr r7, r7, #24\n\t" - "ldrb r0, [r8, r0, lsl #2]\n\t" - "ldrb r1, [r8, r1, lsl #2]\n\t" - "ldrb r4, [r8, r4, lsl #2]\n\t" - "ldrb r7, [r8, r7, lsl #2]\n\t" - "eor r3, r7, r0, lsl #8\n\t" - "eor r3, r3, r1, lsl #16\n\t" - "eor r3, r3, r4, lsl #24\n\t" - "ldm %[ks]!, {r0, r1, r4, r5, r6, r7}\n\t" - "eor r0, r0, r3\n\t" - "ldm lr!, {r3}\n\t" - "eor r0, r0, r3\n\t" - "eor r1, r1, r0\n\t" - "eor r4, r4, r1\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "stm %[ks], {r0, r1, r4, r5, r6, r7}\n\t" - "subs r12, r12, #1\n\t" - "bne L_AES_set_encrypt_key_loop_192_%=\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r0, r7, #24\n\t" - "lsr r0, r0, #24\n\t" -#else - "uxtb r0, r7\n\t" -#endif -#else - "ubfx r0, r7, #0, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r1, r7, #16\n\t" - "lsr r1, r1, #24\n\t" -#else - "uxtb r1, r7, ror #8\n\t" -#endif -#else - "ubfx r1, r7, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r7, #8\n\t" - "lsr r4, r4, #24\n\t" -#else - "uxtb r4, r7, ror #16\n\t" -#endif -#else - "ubfx r4, r7, #16, #8\n\t" -#endif - "lsr r7, r7, #24\n\t" - "ldrb r0, [r8, r0, lsl #2]\n\t" - "ldrb r1, [r8, r1, lsl #2]\n\t" - "ldrb r4, [r8, r4, lsl #2]\n\t" - "ldrb r7, [r8, r7, lsl #2]\n\t" - "eor r3, r7, r0, lsl #8\n\t" - "eor r3, r3, r1, lsl #16\n\t" - "eor r3, r3, r4, lsl #24\n\t" - "ldm %[ks]!, {r0, r1, r4, r5, r6, r7}\n\t" - "eor r0, r0, r3\n\t" - "ldm lr!, {r3}\n\t" - "eor r0, r0, r3\n\t" - "eor r1, r1, r0\n\t" - "eor r4, r4, r1\n\t" - "eor r5, r5, r4\n\t" - "stm %[ks], {r0, r1, r4, r5}\n\t" - "b L_AES_set_encrypt_key_end_%=\n\t" - "\n" - "L_AES_set_encrypt_key_start_128_%=:\n\t" - "ldr r4, [%[key]]\n\t" - "ldr r5, [%[key], #4]\n\t" - "ldr r6, [%[key], #8]\n\t" - "ldr r7, [%[key], #12]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - /* REV r4, r4 */ - "eor r3, r4, r4, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "eor r4, r4, r3, lsr #8\n\t" - /* REV r5, r5 */ - "eor r3, r5, r5, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r5, r5, #8\n\t" - "eor r5, r5, r3, lsr #8\n\t" - /* REV r6, r6 */ - "eor r3, r6, r6, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r6, r6, #8\n\t" - "eor r6, r6, r3, lsr #8\n\t" - /* REV r7, r7 */ - "eor r3, r7, r7, ror #16\n\t" - "bic r3, r3, #0xff0000\n\t" - "ror r7, r7, #8\n\t" - "eor r7, r7, r3, lsr #8\n\t" -#else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "stm %[ks], {r4, r5, r6, r7}\n\t" - "mov r12, #10\n\t" - "\n" - "L_AES_set_encrypt_key_loop_128_%=:\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r7, #24\n\t" - "lsr r4, r4, #24\n\t" -#else - "uxtb r4, r7\n\t" -#endif -#else - "ubfx r4, r7, #0, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r7, #16\n\t" - "lsr r5, r5, #24\n\t" -#else - "uxtb r5, r7, ror #8\n\t" -#endif -#else - "ubfx r5, r7, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r7, #8\n\t" - "lsr r6, r6, #24\n\t" -#else - "uxtb r6, r7, ror #16\n\t" -#endif -#else - "ubfx r6, r7, #16, #8\n\t" -#endif - "lsr r7, r7, #24\n\t" - "ldrb r4, [r8, r4, lsl #2]\n\t" - "ldrb r5, [r8, r5, lsl #2]\n\t" - "ldrb r6, [r8, r6, lsl #2]\n\t" - "ldrb r7, [r8, r7, lsl #2]\n\t" - "eor r3, r7, r4, lsl #8\n\t" - "eor r3, r3, r5, lsl #16\n\t" - "eor r3, r3, r6, lsl #24\n\t" - "ldm %[ks]!, {r4, r5, r6, r7}\n\t" - "eor r4, r4, r3\n\t" - "ldm lr!, {r3}\n\t" - "eor r4, r4, r3\n\t" - "eor r5, r5, r4\n\t" - "eor r6, r6, r5\n\t" - "eor r7, r7, r6\n\t" - "stm %[ks], {r4, r5, r6, r7}\n\t" - "subs r12, r12, #1\n\t" - "bne L_AES_set_encrypt_key_loop_128_%=\n\t" - "\n" - "L_AES_set_encrypt_key_end_%=:\n\t" - "pop {%[L_AES_ARM32_rcon]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [key] "+r" (key), [len] "+r" (len), [ks] "+r" (ks), - [L_AES_ARM32_te] "+r" (L_AES_ARM32_te_c), - [L_AES_ARM32_rcon] "+r" (L_AES_ARM32_rcon_c) - : -#else - : - : [key] "r" (key), [len] "r" (len), [ks] "r" (ks), - [L_AES_ARM32_te] "r" (L_AES_ARM32_te_c), - [L_AES_ARM32_rcon] "r" (L_AES_ARM32_rcon_c) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8" - ); -} - -#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE -void AES_encrypt_block(const word32* te_p, int nr_p, int len_p, - const word32* ks_p); -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te_p, int nr_p, - int len_p, const word32* ks_p) -#else -WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te, int nr, int len, - const word32* ks) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const word32* te __asm__ ("r0") = (const word32*)te_p; - register int nr __asm__ ("r1") = (int)nr_p; - register int len __asm__ ("r2") = (int)len_p; - register const word32* ks __asm__ ("r3") = (const word32*)ks_p; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "\n" - "L_AES_encrypt_block_nr_%=:\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r8, r5, #8\n\t" - "lsr r8, r8, #24\n\t" -#else - "uxtb r8, r5, ror #16\n\t" -#endif -#else - "ubfx r8, r5, #16, #8\n\t" -#endif - "lsr r11, r4, #24\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r6, #16\n\t" - "lsr lr, lr, #24\n\t" -#else - "uxtb lr, r6, ror #8\n\t" -#endif -#else - "ubfx lr, r6, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r7, #24\n\t" - "lsr r2, r2, #24\n\t" -#else - "uxtb r2, r7\n\t" -#endif -#else - "ubfx r2, r7, #0, #8\n\t" -#endif - "ldr r8, [%[te], r8, lsl #2]\n\t" - "ldr r11, [%[te], r11, lsl #2]\n\t" - "ldr lr, [%[te], lr, lsl #2]\n\t" - "ldr r2, [%[te], r2, lsl #2]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r9, r6, #8\n\t" - "lsr r9, r9, #24\n\t" -#else - "uxtb r9, r6, ror #16\n\t" -#endif -#else - "ubfx r9, r6, #16, #8\n\t" -#endif - "eor r8, r8, r11, ror #24\n\t" - "lsr r11, r5, #24\n\t" - "eor r8, r8, lr, ror #8\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r7, #16\n\t" - "lsr lr, lr, #24\n\t" -#else - "uxtb lr, r7, ror #8\n\t" -#endif -#else - "ubfx lr, r7, #8, #8\n\t" -#endif - "eor r8, r8, r2, ror #16\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r4, #24\n\t" - "lsr r2, r2, #24\n\t" -#else - "uxtb r2, r4\n\t" -#endif -#else - "ubfx r2, r4, #0, #8\n\t" -#endif - "ldr r9, [%[te], r9, lsl #2]\n\t" - "ldr r11, [%[te], r11, lsl #2]\n\t" - "ldr lr, [%[te], lr, lsl #2]\n\t" - "ldr r2, [%[te], r2, lsl #2]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r10, r7, #8\n\t" - "lsr r10, r10, #24\n\t" -#else - "uxtb r10, r7, ror #16\n\t" -#endif -#else - "ubfx r10, r7, #16, #8\n\t" -#endif - "eor r9, r9, r11, ror #24\n\t" - "lsr r11, r6, #24\n\t" - "eor r9, r9, lr, ror #8\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r4, #16\n\t" - "lsr lr, lr, #24\n\t" -#else - "uxtb lr, r4, ror #8\n\t" -#endif -#else - "ubfx lr, r4, #8, #8\n\t" -#endif - "eor r9, r9, r2, ror #16\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #24\n\t" - "lsr r2, r2, #24\n\t" -#else - "uxtb r2, r5\n\t" -#endif -#else - "ubfx r2, r5, #0, #8\n\t" -#endif - "ldr r10, [%[te], r10, lsl #2]\n\t" - "ldr r11, [%[te], r11, lsl #2]\n\t" - "ldr lr, [%[te], lr, lsl #2]\n\t" - "ldr r2, [%[te], r2, lsl #2]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r6, #24\n\t" - "lsr r6, r6, #24\n\t" -#else - "uxtb r6, r6\n\t" -#endif -#else - "ubfx r6, r6, #0, #8\n\t" -#endif - "eor r10, r10, r11, ror #24\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r11, r4, #8\n\t" - "lsr r11, r11, #24\n\t" -#else - "uxtb r11, r4, ror #16\n\t" -#endif -#else - "ubfx r11, r4, #16, #8\n\t" -#endif - "eor r10, r10, lr, ror #8\n\t" - "lsr lr, r7, #24\n\t" - "eor r10, r10, r2, ror #16\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #16\n\t" - "lsr r2, r2, #24\n\t" -#else - "uxtb r2, r5, ror #8\n\t" -#endif -#else - "ubfx r2, r5, #8, #8\n\t" -#endif - "ldr r6, [%[te], r6, lsl #2]\n\t" - "ldr lr, [%[te], lr, lsl #2]\n\t" - "ldr r11, [%[te], r11, lsl #2]\n\t" - "ldr r2, [%[te], r2, lsl #2]\n\t" - "eor lr, lr, r6, ror #24\n\t" - "ldm %[ks]!, {r4, r5, r6, r7}\n\t" - "eor r11, r11, lr, ror #24\n\t" - "eor r11, r11, r2, ror #8\n\t" - /* XOR in Key Schedule */ - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r9, #8\n\t" - "lsr r4, r4, #24\n\t" -#else - "uxtb r4, r9, ror #16\n\t" -#endif -#else - "ubfx r4, r9, #16, #8\n\t" -#endif - "lsr r7, r8, #24\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r10, #16\n\t" - "lsr lr, lr, #24\n\t" -#else - "uxtb lr, r10, ror #8\n\t" -#endif -#else - "ubfx lr, r10, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r11, #24\n\t" - "lsr r2, r2, #24\n\t" + "ubfx lr, r10, #8, #8\n\t" +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r11, #24\n\t" + "lsr r2, r2, #24\n\t" #else "uxtb r2, r11\n\t" #endif @@ -24271,222 +15484,713 @@ WC_OMIT_FRAME_POINTER void AES_CBC_decrypt(const unsigned char* in, #endif "eor r5, r5, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r8, #16\n\t" - "lsr r7, r7, #24\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r8, #16\n\t" + "lsr r7, r7, #24\n\t" +#else + "uxtb r7, r8, ror #8\n\t" +#endif +#else + "ubfx r7, r8, #8, #8\n\t" +#endif + "eor r5, r5, r12, lsl #16\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r12, r9, #8\n\t" + "lsr r12, r12, #24\n\t" +#else + "uxtb r12, r9, ror #16\n\t" +#endif +#else + "ubfx r12, r9, #16, #8\n\t" +#endif + "eor r5, r5, lr, lsl #24\n\t" + "lsr lr, r10, #24\n\t" + "ldrb r7, [r2, r7]\n\t" + "ldrb lr, [r2, lr]\n\t" + "ldrb r6, [r2, r6]\n\t" + "ldrb r12, [r2, r12]\n\t" + "lsr r11, r11, #24\n\t" + "eor r6, r6, r7, lsl #8\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r8, #24\n\t" + "lsr r7, r7, #24\n\t" +#else + "uxtb r7, r8\n\t" +#endif +#else + "ubfx r7, r8, #0, #8\n\t" +#endif + "eor r6, r6, r12, lsl #16\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r12, r9, #16\n\t" + "lsr r12, r12, #24\n\t" +#else + "uxtb r12, r9, ror #8\n\t" +#endif +#else + "ubfx r12, r9, #8, #8\n\t" +#endif + "eor r6, r6, lr, lsl #24\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r10, #8\n\t" + "lsr lr, lr, #24\n\t" +#else + "uxtb lr, r10, ror #16\n\t" +#endif +#else + "ubfx lr, r10, #16, #8\n\t" +#endif + "ldrb r11, [r2, r11]\n\t" + "ldrb r12, [r2, r12]\n\t" + "ldrb r7, [r2, r7]\n\t" + "ldrb lr, [r2, lr]\n\t" + "eor r12, r12, r11, lsl #16\n\t" + "ldm %[ks], {r8, r9, r10, r11}\n\t" + "eor r7, r7, r12, lsl #8\n\t" + "eor r7, r7, lr, lsl #16\n\t" + /* XOR in Key Schedule */ + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" +#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ + "ldr lr, [sp, #16]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "eor r8, r4, r4, ror #16\n\t" + "eor r9, r5, r5, ror #16\n\t" + "eor r10, r6, r6, ror #16\n\t" + "eor r11, r7, r7, ror #16\n\t" + "bic r8, r8, #0xff0000\n\t" + "bic r9, r9, #0xff0000\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "ror r5, r5, #8\n\t" + "ror r6, r6, #8\n\t" + "ror r7, r7, #8\n\t" + "eor r4, r4, r8, lsr #8\n\t" + "eor r5, r5, r9, lsr #8\n\t" + "eor r6, r6, r10, lsr #8\n\t" + "eor r7, r7, r11, lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "ldr r8, [lr, #16]\n\t" + "ldr r9, [lr, #20]\n\t" +#else + "ldrd r8, r9, [lr, #16]\n\t" +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "ldr r10, [lr, #24]\n\t" + "ldr r11, [lr, #28]\n\t" +#else + "ldrd r10, r11, [lr, #24]\n\t" +#endif + "pop {r1, r12, lr}\n\t" + "ldr %[ks], [sp]\n\t" + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "str r4, [%[out]]\n\t" + "str r5, [%[out], #4]\n\t" + "str r6, [%[out], #8]\n\t" + "str r7, [%[out], #12]\n\t" + "subs r12, r12, #16\n\t" + "add lr, lr, #16\n\t" + "add %[out], %[out], #16\n\t" + "bne L_AES_CBC_decrypt_loop_block_128_%=\n\t" + "b L_AES_CBC_decrypt_end_%=\n\t" + "\n" + "L_AES_CBC_decrypt_end_odd_%=:\n\t" + "ldr r4, [sp, #4]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "ldr r8, [r4, #16]\n\t" + "ldr r9, [r4, #20]\n\t" +#else + "ldrd r8, r9, [r4, #16]\n\t" +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "ldr r10, [r4, #24]\n\t" + "ldr r11, [r4, #28]\n\t" +#else + "ldrd r10, r11, [r4, #24]\n\t" +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r4, {r8, r9}\n\t" +#else + "strd r8, r9, [r4]\n\t" +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "str r10, [r4, #8]\n\t" + "str r11, [r4, #12]\n\t" +#else + "strd r10, r11, [r4, #8]\n\t" +#endif + "\n" + "L_AES_CBC_decrypt_end_%=:\n\t" + "pop {%[ks]-r4}\n\t" + "pop {%[nr], %[iv]}\n\t" + "pop {%[L_AES_ARM32_td_ecb], %[L_AES_ARM32_cbc_td4]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [len] "+r" (len), [ks] "+r" (ks), + [nr] "+r" (nr), [iv] "+r" (iv), + [L_AES_ARM32_td_ecb] "+r" (L_AES_ARM32_td_ecb_c), + [L_AES_ARM32_cbc_td4] "+r" (L_AES_ARM32_cbc_td4_c) + : +#else + : + : [in] "r" (in), [out] "r" (out), [len] "r" (len), [ks] "r" (ks), + [nr] "r" (nr), [iv] "r" (iv), + [L_AES_ARM32_td_ecb] "r" (L_AES_ARM32_td_ecb_c), + [L_AES_ARM32_cbc_td4] "r" (L_AES_ARM32_cbc_td4_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r6", "r7", "r8", "r9", "r10", "r11" + ); +} + +#endif /* HAVE_AES_CBC */ +#endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER || HAVE_AES_CBC + * HAVE_AES_ECB */ +#endif /* HAVE_AES_DECRYPT */ +#ifdef HAVE_AESGCM +XALIGNED(8) static const word32 L_GCM_gmult_len_r[] = { + 0x00000000, 0x1c200000, 0x38400000, 0x24600000, + 0x70800000, 0x6ca00000, 0x48c00000, 0x54e00000, + 0xe1000000, 0xfd200000, 0xd9400000, 0xc5600000, + 0x91800000, 0x8da00000, 0xa9c00000, 0xb5e00000, +}; + +void GCM_gmult_len(unsigned char* x_p, const unsigned char** m_p, + const unsigned char* data_p, unsigned long len_p); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x_p, + const unsigned char** m_p, const unsigned char* data_p, unsigned long len_p) +#else +WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x, + const unsigned char** m, const unsigned char* data, unsigned long len) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register unsigned char* x __asm__ ("r0") = (unsigned char*)x_p; + register const unsigned char** m __asm__ ("r1") = + (const unsigned char**)m_p; + register const unsigned char* data __asm__ ("r2") = + (const unsigned char*)data_p; + register unsigned long len __asm__ ("r3") = (unsigned long)len_p; + register word32* L_GCM_gmult_len_r_c __asm__ ("r12") = + (word32*)&L_GCM_gmult_len_r; +#else + register word32* L_GCM_gmult_len_r_c = (word32*)&L_GCM_gmult_len_r; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_GCM_gmult_len_r]}\n\t" + "mov lr, %[L_GCM_gmult_len_r]\n\t" + "\n" + "L_GCM_gmult_len_start_block_%=:\n\t" + "push {r3}\n\t" + "ldr r12, [r0, #12]\n\t" + "ldr %[len], [r2, #12]\n\t" + "eor r12, r12, %[len]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl %[len], r12, #4\n\t" + "lsr %[len], %[len], #28\n\t" +#else + "ubfx %[len], r12, #24, #4\n\t" +#endif + "add %[len], %[m], %[len], lsl #4\n\t" + "ldm %[len], {r8, r9, r10, r11}\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsr r4, r12, #28\n\t" #else - "uxtb r7, r8, ror #8\n\t" + "ubfx r4, r12, #28, #4\n\t" #endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r4, r12, #12\n\t" + "lsr r4, r4, #28\n\t" #else - "ubfx r7, r8, #8, #8\n\t" + "ubfx r4, r12, #16, #4\n\t" #endif - "eor r5, r5, r12, lsl #16\n\t" + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r12, r9, #8\n\t" - "lsr r12, r12, #24\n\t" + "lsl r4, r12, #8\n\t" + "lsr r4, r4, #28\n\t" #else - "uxtb r12, r9, ror #16\n\t" + "ubfx r4, r12, #20, #4\n\t" #endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r4, r12, #20\n\t" + "lsr r4, r4, #28\n\t" #else - "ubfx r12, r9, #16, #8\n\t" + "ubfx r4, r12, #8, #4\n\t" #endif - "eor r5, r5, lr, lsl #24\n\t" - "lsr lr, r10, #24\n\t" - "ldrb r7, [r2, r7]\n\t" - "ldrb lr, [r2, lr]\n\t" - "ldrb r6, [r2, r6]\n\t" - "ldrb r12, [r2, r12]\n\t" - "lsr r11, r11, #24\n\t" - "eor r6, r6, r7, lsl #8\n\t" + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r8, #24\n\t" - "lsr r7, r7, #24\n\t" + "lsl r4, r12, #16\n\t" + "lsr r4, r4, #28\n\t" #else - "uxtb r7, r8\n\t" + "ubfx r4, r12, #12, #4\n\t" #endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" + "and r4, r12, #15\n\t" + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r4, r12, #24\n\t" + "lsr r4, r4, #28\n\t" #else - "ubfx r7, r8, #0, #8\n\t" + "ubfx r4, r12, #4, #4\n\t" #endif - "eor r6, r6, r12, lsl #16\n\t" + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "ldr r12, [r0, #8]\n\t" + "ldr %[len], [r2, #8]\n\t" + "eor r12, r12, %[len]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r12, r9, #16\n\t" - "lsr r12, r12, #24\n\t" + "lsl %[len], r12, #4\n\t" + "lsr %[len], %[len], #28\n\t" #else - "uxtb r12, r9, ror #8\n\t" + "ubfx %[len], r12, #24, #4\n\t" #endif + "add %[len], %[m], %[len], lsl #4\n\t" + "ldm %[len], {r4, r5, r6, r7}\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsr r4, r12, #28\n\t" #else - "ubfx r12, r9, #8, #8\n\t" + "ubfx r4, r12, #28, #4\n\t" #endif - "eor r6, r6, lr, lsl #24\n\t" + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r10, #8\n\t" - "lsr lr, lr, #24\n\t" + "lsl r4, r12, #12\n\t" + "lsr r4, r4, #28\n\t" #else - "uxtb lr, r10, ror #16\n\t" + "ubfx r4, r12, #16, #4\n\t" #endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r4, r12, #8\n\t" + "lsr r4, r4, #28\n\t" #else - "ubfx lr, r10, #16, #8\n\t" -#endif - "ldrb r11, [r2, r11]\n\t" - "ldrb r12, [r2, r12]\n\t" - "ldrb r7, [r2, r7]\n\t" - "ldrb lr, [r2, lr]\n\t" - "eor r12, r12, r11, lsl #16\n\t" - "ldm %[ks], {r8, r9, r10, r11}\n\t" - "eor r7, r7, r12, lsl #8\n\t" - "eor r7, r7, lr, lsl #16\n\t" - /* XOR in Key Schedule */ - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" -#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - "ldr lr, [sp, #16]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "eor r8, r4, r4, ror #16\n\t" - "eor r9, r5, r5, ror #16\n\t" - "eor r10, r6, r6, ror #16\n\t" - "eor r11, r7, r7, ror #16\n\t" - "bic r8, r8, #0xff0000\n\t" - "bic r9, r9, #0xff0000\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "ror r5, r5, #8\n\t" - "ror r6, r6, #8\n\t" - "ror r7, r7, #8\n\t" - "eor r4, r4, r8, lsr #8\n\t" - "eor r5, r5, r9, lsr #8\n\t" - "eor r6, r6, r10, lsr #8\n\t" - "eor r7, r7, r11, lsr #8\n\t" + "ubfx r4, r12, #20, #4\n\t" +#endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r4, r12, #20\n\t" + "lsr r4, r4, #28\n\t" #else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "ubfx r4, r12, #8, #4\n\t" +#endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "ldr r8, [lr, #16]\n\t" - "ldr r9, [lr, #20]\n\t" + "lsl r4, r12, #16\n\t" + "lsr r4, r4, #28\n\t" #else - "ldrd r8, r9, [lr, #16]\n\t" + "ubfx r4, r12, #12, #4\n\t" #endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" + "and r4, r12, #15\n\t" + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "ldr r10, [lr, #24]\n\t" - "ldr r11, [lr, #28]\n\t" + "lsl r4, r12, #24\n\t" + "lsr r4, r4, #28\n\t" #else - "ldrd r10, r11, [lr, #24]\n\t" + "ubfx r4, r12, #4, #4\n\t" #endif - "pop {r1, r12, lr}\n\t" - "ldr %[ks], [sp]\n\t" - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" - "str r4, [%[out]]\n\t" - "str r5, [%[out], #4]\n\t" - "str r6, [%[out], #8]\n\t" - "str r7, [%[out], #12]\n\t" - "subs r12, r12, #16\n\t" - "add lr, lr, #16\n\t" - "add %[out], %[out], #16\n\t" - "bne L_AES_CBC_decrypt_loop_block_128_%=\n\t" - "b L_AES_CBC_decrypt_end_%=\n\t" - "\n" - "L_AES_CBC_decrypt_end_odd_%=:\n\t" - "ldr r4, [sp, #4]\n\t" + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "ldr r12, [r0, #4]\n\t" + "ldr %[len], [r2, #4]\n\t" + "eor r12, r12, %[len]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "ldr r8, [r4, #16]\n\t" - "ldr r9, [r4, #20]\n\t" + "lsl %[len], r12, #4\n\t" + "lsr %[len], %[len], #28\n\t" #else - "ldrd r8, r9, [r4, #16]\n\t" + "ubfx %[len], r12, #24, #4\n\t" #endif + "add %[len], %[m], %[len], lsl #4\n\t" + "ldm %[len], {r4, r5, r6, r7}\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "ldr r10, [r4, #24]\n\t" - "ldr r11, [r4, #28]\n\t" + "lsr r4, r12, #28\n\t" #else - "ldrd r10, r11, [r4, #24]\n\t" + "ubfx r4, r12, #28, #4\n\t" #endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "stm r4, {r8, r9}\n\t" + "lsl r4, r12, #12\n\t" + "lsr r4, r4, #28\n\t" #else - "strd r8, r9, [r4]\n\t" + "ubfx r4, r12, #16, #4\n\t" #endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "str r10, [r4, #8]\n\t" - "str r11, [r4, #12]\n\t" + "lsl r4, r12, #8\n\t" + "lsr r4, r4, #28\n\t" #else - "strd r10, r11, [r4, #8]\n\t" + "ubfx r4, r12, #20, #4\n\t" #endif - "\n" - "L_AES_CBC_decrypt_end_%=:\n\t" - "pop {%[ks]-r4}\n\t" - "pop {%[nr], %[iv]}\n\t" - "pop {%[L_AES_ARM32_td_ecb], %[L_AES_ARM32_cbc_td4]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [len] "+r" (len), [ks] "+r" (ks), - [nr] "+r" (nr), [iv] "+r" (iv), - [L_AES_ARM32_td_ecb] "+r" (L_AES_ARM32_td_ecb_c), - [L_AES_ARM32_cbc_td4] "+r" (L_AES_ARM32_cbc_td4_c) - : + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r4, r12, #20\n\t" + "lsr r4, r4, #28\n\t" #else - : - : [in] "r" (in), [out] "r" (out), [len] "r" (len), [ks] "r" (ks), - [nr] "r" (nr), [iv] "r" (iv), - [L_AES_ARM32_td_ecb] "r" (L_AES_ARM32_td_ecb_c), - [L_AES_ARM32_cbc_td4] "r" (L_AES_ARM32_cbc_td4_c) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r6", "r7", "r8", "r9", "r10", "r11" - ); -} - -#endif /* HAVE_AES_CBC */ -#endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER || HAVE_AES_CBC - * HAVE_AES_ECB */ -#endif /* HAVE_AES_DECRYPT */ -#ifdef HAVE_AESGCM -XALIGNED(8) static const word32 L_GCM_gmult_len_r[] = { - 0x00000000, 0x1c200000, 0x38400000, 0x24600000, - 0x70800000, 0x6ca00000, 0x48c00000, 0x54e00000, - 0xe1000000, 0xfd200000, 0xd9400000, 0xc5600000, - 0x91800000, 0x8da00000, 0xa9c00000, 0xb5e00000, -}; - -void GCM_gmult_len(unsigned char* x_p, const unsigned char** m_p, - const unsigned char* data_p, unsigned long len_p); -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x_p, - const unsigned char** m_p, const unsigned char* data_p, unsigned long len_p) + "ubfx r4, r12, #8, #4\n\t" +#endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r4, r12, #16\n\t" + "lsr r4, r4, #28\n\t" #else -WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x, - const unsigned char** m, const unsigned char* data, unsigned long len) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register unsigned char* x __asm__ ("r0") = (unsigned char*)x_p; - register const unsigned char** m __asm__ ("r1") = - (const unsigned char**)m_p; - register const unsigned char* data __asm__ ("r2") = - (const unsigned char*)data_p; - register unsigned long len __asm__ ("r3") = (unsigned long)len_p; - register word32* L_GCM_gmult_len_r_c __asm__ ("r12") = - (word32*)&L_GCM_gmult_len_r; + "ubfx r4, r12, #12, #4\n\t" +#endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" + "and r4, r12, #15\n\t" + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r4, r12, #24\n\t" + "lsr r4, r4, #28\n\t" #else - register word32* L_GCM_gmult_len_r_c = (word32*)&L_GCM_gmult_len_r; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[L_GCM_gmult_len_r]}\n\t" - "mov lr, %[L_GCM_gmult_len_r]\n\t" - "\n" - "L_GCM_gmult_len_start_block_%=:\n\t" - "push {r3}\n\t" - "ldr r12, [r0, #12]\n\t" - "ldr %[len], [r2, #12]\n\t" + "ubfx r4, r12, #4, #4\n\t" +#endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "lsr r6, r10, #4\n\t" + "and %[len], r11, #15\n\t" + "lsr r11, r11, #4\n\t" + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "ldr r12, [r0]\n\t" + "ldr %[len], [r2]\n\t" "eor r12, r12, %[len]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) "lsl %[len], r12, #4\n\t" @@ -24495,7 +16199,11 @@ WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x, "ubfx %[len], r12, #24, #4\n\t" #endif "add %[len], %[m], %[len], lsl #4\n\t" - "ldm %[len], {r8, r9, r10, r11}\n\t" + "ldm %[len], {r4, r5, r6, r7}\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" "lsr r6, r10, #4\n\t" "and %[len], r11, #15\n\t" "lsr r11, r11, #4\n\t" @@ -24623,646 +16331,469 @@ WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x, "lsl r4, r12, #24\n\t" "lsr r4, r4, #28\n\t" #else - "ubfx r4, r12, #4, #4\n\t" + "ubfx r4, r12, #4, #4\n\t" +#endif + "eor r11, r11, r10, lsl #28\n\t" + "ldr %[len], [lr, r3, lsl #2]\n\t" + "add r4, %[m], r4, lsl #4\n\t" + "eor r10, r6, r9, lsl #28\n\t" + "lsr r9, r9, #4\n\t" + "ldm r4, {r4, r5, r6, r7}\n\t" + "eor r9, r9, r8, lsl #28\n\t" + "eor r8, %[len], r8, lsr #4\n\t" + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r8, r8 */ + "eor %[len], r8, r8, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r8, r8, #8\n\t" + "eor r8, r8, %[len], lsr #8\n\t" + /* REV r9, r9 */ + "eor %[len], r9, r9, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r9, r9, #8\n\t" + "eor r9, r9, %[len], lsr #8\n\t" + /* REV r10, r10 */ + "eor %[len], r10, r10, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r10, r10, #8\n\t" + "eor r10, r10, %[len], lsr #8\n\t" + /* REV r11, r11 */ + "eor %[len], r11, r11, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r11, r11, #8\n\t" + "eor r11, r11, %[len], lsr #8\n\t" +#else + "rev r8, r8\n\t" + "rev r9, r9\n\t" + "rev r10, r10\n\t" + "rev r11, r11\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "stm %[x], {r8, r9, r10, r11}\n\t" + "pop {r3}\n\t" + "subs %[len], %[len], #16\n\t" + "add %[data], %[data], #16\n\t" + "bne L_GCM_gmult_len_start_block_%=\n\t" + "pop {%[L_GCM_gmult_len_r]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [x] "+r" (x), [m] "+r" (m), [data] "+r" (data), [len] "+r" (len), + [L_GCM_gmult_len_r] "+r" (L_GCM_gmult_len_r_c) + : +#else + : + : [x] "r" (x), [m] "r" (m), [data] "r" (data), [len] "r" (len), + [L_GCM_gmult_len_r] "r" (L_GCM_gmult_len_r_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "r9", "r10", + "r11" + ); +} + +static const word32* L_AES_ARM32_te_gcm = L_AES_ARM32_te_data; +void AES_GCM_encrypt(const unsigned char* in_p, unsigned char* out_p, + unsigned long len_p, const unsigned char* ks_p, int nr_p, + unsigned char* ctr_p); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_GCM_encrypt(const unsigned char* in_p, + unsigned char* out_p, unsigned long len_p, const unsigned char* ks_p, + int nr_p, unsigned char* ctr_p) +#else +WC_OMIT_FRAME_POINTER void AES_GCM_encrypt(const unsigned char* in, + unsigned char* out, unsigned long len, const unsigned char* ks, int nr, + unsigned char* ctr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const unsigned char* in __asm__ ("r0") = + (const unsigned char*)in_p; + register unsigned char* out __asm__ ("r1") = (unsigned char*)out_p; + register unsigned long len __asm__ ("r2") = (unsigned long)len_p; + register const unsigned char* ks __asm__ ("r3") = + (const unsigned char*)ks_p; + register int nr __asm__ ("r12") = (int)nr_p; + register unsigned char* ctr __asm__ ("lr") = (unsigned char*)ctr_p; + register word32* L_AES_ARM32_te_gcm_c __asm__ ("r4") = + (word32*)L_AES_ARM32_te_gcm; +#else + register word32* L_AES_ARM32_te_gcm_c = (word32*)L_AES_ARM32_te_gcm; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_ARM32_te_gcm]}\n\t" + "push {%[nr], %[ctr]}\n\t" + "ldr r12, [sp]\n\t" + "ldr r8, [sp, #4]\n\t" + "mov lr, %[in]\n\t" + "ldr r0, [sp, #8]\n\t" + "ldm r8, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "eor r10, r4, r4, ror #16\n\t" + "eor r11, r5, r5, ror #16\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "ror r5, r5, #8\n\t" + "eor r4, r4, r10, lsr #8\n\t" + "eor r5, r5, r11, lsr #8\n\t" + "eor r10, r6, r6, ror #16\n\t" + "eor r11, r7, r7, ror #16\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "ror r7, r7, #8\n\t" + "eor r6, r6, r10, lsr #8\n\t" + "eor r7, r7, r11, lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "stm r8, {r4, r5, r6, r7}\n\t" + "push {%[ks], r8}\n\t" + "cmp r12, #10\n\t" + "beq L_AES_GCM_encrypt_start_block_128_%=\n\t" + "cmp r12, #12\n\t" + "beq L_AES_GCM_encrypt_start_block_192_%=\n\t" + "\n" + "L_AES_GCM_encrypt_loop_block_256_%=:\n\t" + "push {r1, %[len], lr}\n\t" + "ldr lr, [sp, #16]\n\t" + "add r7, r7, #1\n\t" + "ldm %[ks]!, {r8, r9, r10, r11}\n\t" + "str r7, [lr, #12]\n\t" + /* Round: 0 - XOR in key schedule */ + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "mov r1, #6\n\t" +#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE + "bl AES_encrypt_block\n\t" +#else + "\n" + "L_AES_GCM_encrypt_block_nr_256_%=:\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r8, r5, #8\n\t" + "lsr r8, r8, #24\n\t" +#else + "uxtb r8, r5, ror #16\n\t" +#endif +#else + "ubfx r8, r5, #16, #8\n\t" +#endif + "lsr r11, r4, #24\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r6, #16\n\t" + "lsr lr, lr, #24\n\t" +#else + "uxtb lr, r6, ror #8\n\t" +#endif +#else + "ubfx lr, r6, #8, #8\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "ldr r12, [r0, #8]\n\t" - "ldr %[len], [r2, #8]\n\t" - "eor r12, r12, %[len]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl %[len], r12, #4\n\t" - "lsr %[len], %[len], #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r7, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx %[len], r12, #24, #4\n\t" + "uxtb r2, r7\n\t" #endif - "add %[len], %[m], %[len], lsl #4\n\t" - "ldm %[len], {r4, r5, r6, r7}\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" +#else + "ubfx r2, r7, #0, #8\n\t" +#endif + "ldr r8, [r0, r8, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsr r4, r12, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r9, r6, #8\n\t" + "lsr r9, r9, #24\n\t" #else - "ubfx r4, r12, #28, #4\n\t" + "uxtb r9, r6, ror #16\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" +#else + "ubfx r9, r6, #16, #8\n\t" +#endif + "eor r8, r8, r11, ror #24\n\t" + "lsr r11, r5, #24\n\t" + "eor r8, r8, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #12\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r7, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r4, r12, #16, #4\n\t" + "uxtb lr, r7, ror #8\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" +#else + "ubfx lr, r7, #8, #8\n\t" +#endif + "eor r8, r8, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #8\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r4, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r4, r12, #20, #4\n\t" + "uxtb r2, r4\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" +#else + "ubfx r2, r4, #0, #8\n\t" +#endif + "ldr r9, [r0, r9, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #20\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r10, r7, #8\n\t" + "lsr r10, r10, #24\n\t" #else - "ubfx r4, r12, #8, #4\n\t" + "uxtb r10, r7, ror #16\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" +#else + "ubfx r10, r7, #16, #8\n\t" +#endif + "eor r9, r9, r11, ror #24\n\t" + "lsr r11, r6, #24\n\t" + "eor r9, r9, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #16\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r4, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r4, r12, #12, #4\n\t" + "uxtb lr, r4, ror #8\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" - "and r4, r12, #15\n\t" - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" +#else + "ubfx lr, r4, #8, #8\n\t" +#endif + "eor r9, r9, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #24\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r5, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r4, r12, #4, #4\n\t" + "uxtb r2, r5\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "ldr r12, [r0, #4]\n\t" - "ldr %[len], [r2, #4]\n\t" - "eor r12, r12, %[len]\n\t" +#else + "ubfx r2, r5, #0, #8\n\t" +#endif + "ldr r10, [r0, r10, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl %[len], r12, #4\n\t" - "lsr %[len], %[len], #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r6, #24\n\t" + "lsr r6, r6, #24\n\t" #else - "ubfx %[len], r12, #24, #4\n\t" + "uxtb r6, r6\n\t" +#endif +#else + "ubfx r6, r6, #0, #8\n\t" #endif - "add %[len], %[m], %[len], lsl #4\n\t" - "ldm %[len], {r4, r5, r6, r7}\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" + "eor r10, r10, r11, ror #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsr r4, r12, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r11, r4, #8\n\t" + "lsr r11, r11, #24\n\t" #else - "ubfx r4, r12, #28, #4\n\t" + "uxtb r11, r4, ror #16\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" +#else + "ubfx r11, r4, #16, #8\n\t" +#endif + "eor r10, r10, lr, ror #8\n\t" + "lsr lr, r7, #24\n\t" + "eor r10, r10, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #12\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r5, #16\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r4, r12, #16, #4\n\t" + "uxtb r2, r5, ror #8\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" +#else + "ubfx r2, r5, #8, #8\n\t" +#endif + "ldr r6, [r0, r6, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r6, ror #24\n\t" + "ldm %[ks]!, {r4, r5, r6, r7}\n\t" + "eor r11, r11, lr, ror #24\n\t" + "eor r11, r11, r2, ror #8\n\t" + /* XOR in Key Schedule */ "eor r8, r8, r4\n\t" "eor r9, r9, r5\n\t" "eor r10, r10, r6\n\t" "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #8\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r9, #8\n\t" + "lsr r4, r4, #24\n\t" #else - "ubfx r4, r12, #20, #4\n\t" + "uxtb r4, r9, ror #16\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" +#else + "ubfx r4, r9, #16, #8\n\t" +#endif + "lsr r7, r8, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #20\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r10, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r4, r12, #8, #4\n\t" + "uxtb lr, r10, ror #8\n\t" +#endif +#else + "ubfx lr, r10, #8, #8\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #16\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r11, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r4, r12, #12, #4\n\t" + "uxtb r2, r11\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" - "and r4, r12, #15\n\t" - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" +#else + "ubfx r2, r11, #0, #8\n\t" +#endif + "ldr r4, [r0, r4, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #24\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r5, r10, #8\n\t" + "lsr r5, r5, #24\n\t" #else - "ubfx r4, r12, #4, #4\n\t" + "uxtb r5, r10, ror #16\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "ldr r12, [r0]\n\t" - "ldr %[len], [r2]\n\t" - "eor r12, r12, %[len]\n\t" +#else + "ubfx r5, r10, #16, #8\n\t" +#endif + "eor r4, r4, r7, ror #24\n\t" + "lsr r7, r9, #24\n\t" + "eor r4, r4, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl %[len], r12, #4\n\t" - "lsr %[len], %[len], #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r11, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx %[len], r12, #24, #4\n\t" + "uxtb lr, r11, ror #8\n\t" #endif - "add %[len], %[m], %[len], lsl #4\n\t" - "ldm %[len], {r4, r5, r6, r7}\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" +#else + "ubfx lr, r11, #8, #8\n\t" +#endif + "eor r4, r4, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsr r4, r12, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r8, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r4, r12, #28, #4\n\t" + "uxtb r2, r8\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" +#else + "ubfx r2, r8, #0, #8\n\t" +#endif + "ldr r5, [r0, r5, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #12\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r11, #8\n\t" + "lsr r6, r6, #24\n\t" +#else + "uxtb r6, r11, ror #16\n\t" +#endif #else - "ubfx r4, r12, #16, #4\n\t" + "ubfx r6, r11, #16, #8\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" + "eor r5, r5, r7, ror #24\n\t" + "lsr r7, r10, #24\n\t" + "eor r5, r5, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #8\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r8, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r4, r12, #20, #4\n\t" + "uxtb lr, r8, ror #8\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #20\n\t" - "lsr r4, r4, #28\n\t" #else - "ubfx r4, r12, #8, #4\n\t" + "ubfx lr, r8, #8, #8\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" + "eor r5, r5, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #16\n\t" - "lsr r4, r4, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r9, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r4, r12, #12, #4\n\t" + "uxtb r2, r9\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" - "and r4, r12, #15\n\t" - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" - "lsr r6, r10, #4\n\t" - "and %[len], r11, #15\n\t" - "lsr r11, r11, #4\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r4, r12, #24\n\t" - "lsr r4, r4, #28\n\t" #else - "ubfx r4, r12, #4, #4\n\t" + "ubfx r2, r9, #0, #8\n\t" #endif - "eor r11, r11, r10, lsl #28\n\t" - "ldr %[len], [lr, r3, lsl #2]\n\t" - "add r4, %[m], r4, lsl #4\n\t" - "eor r10, r6, r9, lsl #28\n\t" - "lsr r9, r9, #4\n\t" - "ldm r4, {r4, r5, r6, r7}\n\t" - "eor r9, r9, r8, lsl #28\n\t" - "eor r8, %[len], r8, lsr #4\n\t" - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" + "ldr r6, [r0, r6, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - /* REV r8, r8 */ - "eor %[len], r8, r8, ror #16\n\t" - "bic %[len], %[len], #0xff0000\n\t" - "ror r8, r8, #8\n\t" - "eor r8, r8, %[len], lsr #8\n\t" - /* REV r9, r9 */ - "eor %[len], r9, r9, ror #16\n\t" - "bic %[len], %[len], #0xff0000\n\t" - "ror r9, r9, #8\n\t" - "eor r9, r9, %[len], lsr #8\n\t" - /* REV r10, r10 */ - "eor %[len], r10, r10, ror #16\n\t" - "bic %[len], %[len], #0xff0000\n\t" - "ror r10, r10, #8\n\t" - "eor r10, r10, %[len], lsr #8\n\t" - /* REV r11, r11 */ - "eor %[len], r11, r11, ror #16\n\t" - "bic %[len], %[len], #0xff0000\n\t" - "ror r11, r11, #8\n\t" - "eor r11, r11, %[len], lsr #8\n\t" + "lsl r10, r10, #24\n\t" + "lsr r10, r10, #24\n\t" #else - "rev r8, r8\n\t" - "rev r9, r9\n\t" - "rev r10, r10\n\t" - "rev r11, r11\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "stm %[x], {r8, r9, r10, r11}\n\t" - "pop {r3}\n\t" - "subs %[len], %[len], #16\n\t" - "add %[data], %[data], #16\n\t" - "bne L_GCM_gmult_len_start_block_%=\n\t" - "pop {%[L_GCM_gmult_len_r]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [x] "+r" (x), [m] "+r" (m), [data] "+r" (data), [len] "+r" (len), - [L_GCM_gmult_len_r] "+r" (L_GCM_gmult_len_r_c) - : + "uxtb r10, r10\n\t" +#endif #else - : - : [x] "r" (x), [m] "r" (m), [data] "r" (data), [len] "r" (len), - [L_GCM_gmult_len_r] "r" (L_GCM_gmult_len_r_c) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "r9", "r10", - "r11" - ); -} - -static const word32* L_AES_ARM32_te_gcm = L_AES_ARM32_te_data; -void AES_GCM_encrypt(const unsigned char* in_p, unsigned char* out_p, - unsigned long len_p, const unsigned char* ks_p, int nr_p, - unsigned char* ctr_p); -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_GCM_encrypt(const unsigned char* in_p, - unsigned char* out_p, unsigned long len_p, const unsigned char* ks_p, - int nr_p, unsigned char* ctr_p) + "ubfx r10, r10, #0, #8\n\t" +#endif + "eor r6, r6, r7, ror #24\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r8, #8\n\t" + "lsr r7, r7, #24\n\t" #else -WC_OMIT_FRAME_POINTER void AES_GCM_encrypt(const unsigned char* in, - unsigned char* out, unsigned long len, const unsigned char* ks, int nr, - unsigned char* ctr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const unsigned char* in __asm__ ("r0") = - (const unsigned char*)in_p; - register unsigned char* out __asm__ ("r1") = (unsigned char*)out_p; - register unsigned long len __asm__ ("r2") = (unsigned long)len_p; - register const unsigned char* ks __asm__ ("r3") = - (const unsigned char*)ks_p; - register int nr __asm__ ("r12") = (int)nr_p; - register unsigned char* ctr __asm__ ("lr") = (unsigned char*)ctr_p; - register word32* L_AES_ARM32_te_gcm_c __asm__ ("r4") = - (word32*)L_AES_ARM32_te_gcm; + "uxtb r7, r8, ror #16\n\t" +#endif #else - register word32* L_AES_ARM32_te_gcm_c = (word32*)L_AES_ARM32_te_gcm; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[L_AES_ARM32_te_gcm]}\n\t" - "push {%[nr], %[ctr]}\n\t" - "ldr r12, [sp]\n\t" - "ldr r8, [sp, #4]\n\t" - "mov lr, %[in]\n\t" - "ldr r0, [sp, #8]\n\t" - "ldm r8, {r4, r5, r6, r7}\n\t" + "ubfx r7, r8, #16, #8\n\t" +#endif + "eor r6, r6, lr, ror #8\n\t" + "lsr lr, r11, #24\n\t" + "eor r6, r6, r2, ror #16\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "eor r10, r4, r4, ror #16\n\t" - "eor r11, r5, r5, ror #16\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "ror r5, r5, #8\n\t" - "eor r4, r4, r10, lsr #8\n\t" - "eor r5, r5, r11, lsr #8\n\t" - "eor r10, r6, r6, ror #16\n\t" - "eor r11, r7, r7, ror #16\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r6, r6, #8\n\t" - "ror r7, r7, #8\n\t" - "eor r6, r6, r10, lsr #8\n\t" - "eor r7, r7, r11, lsr #8\n\t" + "lsl r2, r9, #16\n\t" + "lsr r2, r2, #24\n\t" #else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "stm r8, {r4, r5, r6, r7}\n\t" - "push {%[ks], r8}\n\t" - "cmp r12, #10\n\t" - "beq L_AES_GCM_encrypt_start_block_128_%=\n\t" - "cmp r12, #12\n\t" - "beq L_AES_GCM_encrypt_start_block_192_%=\n\t" - "\n" - "L_AES_GCM_encrypt_loop_block_256_%=:\n\t" - "push {r1, %[len], lr}\n\t" - "ldr lr, [sp, #16]\n\t" - "add r7, r7, #1\n\t" + "uxtb r2, r9, ror #8\n\t" +#endif +#else + "ubfx r2, r9, #8, #8\n\t" +#endif + "ldr r10, [r0, r10, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r10, ror #24\n\t" "ldm %[ks]!, {r8, r9, r10, r11}\n\t" - "str r7, [lr, #12]\n\t" - /* Round: 0 - XOR in key schedule */ + "eor r7, r7, lr, ror #24\n\t" + "eor r7, r7, r2, ror #8\n\t" + /* XOR in Key Schedule */ "eor r4, r4, r8\n\t" "eor r5, r5, r9\n\t" "eor r6, r6, r10\n\t" "eor r7, r7, r11\n\t" - "mov r1, #6\n\t" -#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE - "bl AES_encrypt_block\n\t" -#else - "\n" - "L_AES_GCM_encrypt_block_nr_256_%=:\n\t" + "subs r1, r1, #1\n\t" + "bne L_AES_GCM_encrypt_block_nr_256_%=\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) "lsl r8, r5, #8\n\t" @@ -25423,164 +16954,227 @@ WC_OMIT_FRAME_POINTER void AES_GCM_encrypt(const unsigned char* in, "eor r11, r11, r7\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r9, #8\n\t" + "lsl r4, r11, #24\n\t" "lsr r4, r4, #24\n\t" #else - "uxtb r4, r9, ror #16\n\t" + "uxtb r4, r11\n\t" #endif #else - "ubfx r4, r9, #16, #8\n\t" + "ubfx r4, r11, #0, #8\n\t" #endif - "lsr r7, r8, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r10, #16\n\t" - "lsr lr, lr, #24\n\t" + "lsl r7, r10, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb lr, r10, ror #8\n\t" + "uxtb r7, r10, ror #8\n\t" #endif #else - "ubfx lr, r10, #8, #8\n\t" + "ubfx r7, r10, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r11, #24\n\t" - "lsr r2, r2, #24\n\t" + "lsl lr, r9, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r2, r11\n\t" + "uxtb lr, r9, ror #16\n\t" #endif #else - "ubfx r2, r11, #0, #8\n\t" + "ubfx lr, r9, #16, #8\n\t" #endif - "ldr r4, [r0, r4, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "lsr r2, r8, #24\n\t" + "ldrb r4, [r0, r4, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r10, #8\n\t" + "lsl r5, r8, #24\n\t" "lsr r5, r5, #24\n\t" #else - "uxtb r5, r10, ror #16\n\t" + "uxtb r5, r8\n\t" #endif #else - "ubfx r5, r10, #16, #8\n\t" + "ubfx r5, r8, #0, #8\n\t" #endif - "eor r4, r4, r7, ror #24\n\t" - "lsr r7, r9, #24\n\t" - "eor r4, r4, lr, ror #8\n\t" + "eor r4, r4, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r11, #16\n\t" - "lsr lr, lr, #24\n\t" + "lsl r7, r11, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb lr, r11, ror #8\n\t" + "uxtb r7, r11, ror #8\n\t" #endif #else - "ubfx lr, r11, #8, #8\n\t" + "ubfx r7, r11, #8, #8\n\t" #endif - "eor r4, r4, r2, ror #16\n\t" + "eor r4, r4, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r8, #24\n\t" - "lsr r2, r2, #24\n\t" + "lsl lr, r10, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r2, r8\n\t" + "uxtb lr, r10, ror #16\n\t" #endif #else - "ubfx r2, r8, #0, #8\n\t" + "ubfx lr, r10, #16, #8\n\t" #endif - "ldr r5, [r0, r5, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "eor r4, r4, r2, lsl #24\n\t" + "lsr r2, r9, #24\n\t" + "ldrb r5, [r0, r5, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r11, #8\n\t" + "lsl r6, r9, #24\n\t" "lsr r6, r6, #24\n\t" #else - "uxtb r6, r11, ror #16\n\t" + "uxtb r6, r9\n\t" #endif #else - "ubfx r6, r11, #16, #8\n\t" + "ubfx r6, r9, #0, #8\n\t" #endif - "eor r5, r5, r7, ror #24\n\t" - "lsr r7, r10, #24\n\t" - "eor r5, r5, lr, ror #8\n\t" + "eor r5, r5, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r8, #16\n\t" - "lsr lr, lr, #24\n\t" + "lsl r7, r8, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb lr, r8, ror #8\n\t" + "uxtb r7, r8, ror #8\n\t" #endif #else - "ubfx lr, r8, #8, #8\n\t" + "ubfx r7, r8, #8, #8\n\t" #endif - "eor r5, r5, r2, ror #16\n\t" + "eor r5, r5, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r9, #24\n\t" - "lsr r2, r2, #24\n\t" + "lsl lr, r11, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r2, r9\n\t" + "uxtb lr, r11, ror #16\n\t" #endif #else - "ubfx r2, r9, #0, #8\n\t" + "ubfx lr, r11, #16, #8\n\t" #endif - "ldr r6, [r0, r6, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "eor r5, r5, r2, lsl #24\n\t" + "lsr r2, r10, #24\n\t" + "ldrb r6, [r0, r6, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" + "lsr r11, r11, #24\n\t" + "eor r6, r6, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r10, r10, #24\n\t" - "lsr r10, r10, #24\n\t" + "lsl r7, r10, #24\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb r10, r10\n\t" + "uxtb r7, r10\n\t" #endif #else - "ubfx r10, r10, #0, #8\n\t" + "ubfx r7, r10, #0, #8\n\t" #endif - "eor r6, r6, r7, ror #24\n\t" + "eor r6, r6, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r8, #8\n\t" - "lsr r7, r7, #24\n\t" + "lsl lr, r9, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r7, r8, ror #16\n\t" + "uxtb lr, r9, ror #8\n\t" #endif #else - "ubfx r7, r8, #16, #8\n\t" + "ubfx lr, r9, #8, #8\n\t" #endif - "eor r6, r6, lr, ror #8\n\t" - "lsr lr, r11, #24\n\t" - "eor r6, r6, r2, ror #16\n\t" + "eor r6, r6, r2, lsl #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r9, #16\n\t" + "lsl r2, r8, #8\n\t" "lsr r2, r2, #24\n\t" #else - "uxtb r2, r9, ror #8\n\t" + "uxtb r2, r8, ror #16\n\t" #endif #else - "ubfx r2, r9, #8, #8\n\t" + "ubfx r2, r8, #16, #8\n\t" #endif - "ldr r10, [r0, r10, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r10, ror #24\n\t" - "ldm %[ks]!, {r8, r9, r10, r11}\n\t" - "eor r7, r7, lr, ror #24\n\t" - "eor r7, r7, r2, ror #8\n\t" + "ldrb r11, [r0, r11, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r11, lsl #16\n\t" + "ldm %[ks], {r8, r9, r10, r11}\n\t" + "eor r7, r7, lr, lsl #8\n\t" + "eor r7, r7, r2, lsl #16\n\t" /* XOR in Key Schedule */ "eor r4, r4, r8\n\t" "eor r5, r5, r9\n\t" "eor r6, r6, r10\n\t" "eor r7, r7, r11\n\t" - "subs r1, r1, #1\n\t" - "bne L_AES_GCM_encrypt_block_nr_256_%=\n\t" +#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ + "pop {r1, %[len], lr}\n\t" + "ldr %[ks], [sp]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "eor r8, r4, r4, ror #16\n\t" + "eor r9, r5, r5, ror #16\n\t" + "eor r10, r6, r6, ror #16\n\t" + "eor r11, r7, r7, ror #16\n\t" + "bic r8, r8, #0xff0000\n\t" + "bic r9, r9, #0xff0000\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "ror r5, r5, #8\n\t" + "ror r6, r6, #8\n\t" + "ror r7, r7, #8\n\t" + "eor r4, r4, r8, lsr #8\n\t" + "eor r5, r5, r9, lsr #8\n\t" + "eor r6, r6, r10, lsr #8\n\t" + "eor r7, r7, r11, lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "ldr r8, [lr]\n\t" + "ldr r9, [lr, #4]\n\t" + "ldr r10, [lr, #8]\n\t" + "ldr r11, [lr, #12]\n\t" + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "ldr r8, [sp, #4]\n\t" + "str r4, [%[out]]\n\t" + "str r5, [%[out], #4]\n\t" + "str r6, [%[out], #8]\n\t" + "str r7, [%[out], #12]\n\t" + "ldm r8, {r4, r5, r6, r7}\n\t" + "subs %[len], %[len], #16\n\t" + "add lr, lr, #16\n\t" + "add %[out], %[out], #16\n\t" + "bne L_AES_GCM_encrypt_loop_block_256_%=\n\t" + "b L_AES_GCM_encrypt_end_%=\n\t" + "\n" + "L_AES_GCM_encrypt_start_block_192_%=:\n\t" + "\n" + "L_AES_GCM_encrypt_loop_block_192_%=:\n\t" + "push {r1, %[len], lr}\n\t" + "ldr lr, [sp, #16]\n\t" + "add r7, r7, #1\n\t" + "ldm %[ks]!, {r8, r9, r10, r11}\n\t" + "str r7, [lr, #12]\n\t" + /* Round: 0 - XOR in key schedule */ + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "mov r1, #5\n\t" +#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE + "bl AES_encrypt_block\n\t" +#else + "\n" + "L_AES_GCM_encrypt_block_nr_192_%=:\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) "lsl r8, r5, #8\n\t" @@ -25741,227 +17335,164 @@ WC_OMIT_FRAME_POINTER void AES_GCM_encrypt(const unsigned char* in, "eor r11, r11, r7\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r11, #24\n\t" + "lsl r4, r9, #8\n\t" "lsr r4, r4, #24\n\t" #else - "uxtb r4, r11\n\t" + "uxtb r4, r9, ror #16\n\t" #endif #else - "ubfx r4, r11, #0, #8\n\t" + "ubfx r4, r9, #16, #8\n\t" #endif + "lsr r7, r8, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r10, #16\n\t" - "lsr r7, r7, #24\n\t" + "lsl lr, r10, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r7, r10, ror #8\n\t" + "uxtb lr, r10, ror #8\n\t" #endif #else - "ubfx r7, r10, #8, #8\n\t" + "ubfx lr, r10, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r9, #8\n\t" - "lsr lr, lr, #24\n\t" + "lsl r2, r11, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "uxtb lr, r9, ror #16\n\t" + "uxtb r2, r11\n\t" #endif #else - "ubfx lr, r9, #16, #8\n\t" + "ubfx r2, r11, #0, #8\n\t" #endif - "lsr r2, r8, #24\n\t" - "ldrb r4, [r0, r4, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" + "ldr r4, [r0, r4, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r8, #24\n\t" + "lsl r5, r10, #8\n\t" "lsr r5, r5, #24\n\t" #else - "uxtb r5, r8\n\t" + "uxtb r5, r10, ror #16\n\t" #endif #else - "ubfx r5, r8, #0, #8\n\t" + "ubfx r5, r10, #16, #8\n\t" #endif - "eor r4, r4, r7, lsl #8\n\t" + "eor r4, r4, r7, ror #24\n\t" + "lsr r7, r9, #24\n\t" + "eor r4, r4, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r11, #16\n\t" - "lsr r7, r7, #24\n\t" + "lsl lr, r11, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r7, r11, ror #8\n\t" + "uxtb lr, r11, ror #8\n\t" #endif #else - "ubfx r7, r11, #8, #8\n\t" + "ubfx lr, r11, #8, #8\n\t" #endif - "eor r4, r4, lr, lsl #16\n\t" + "eor r4, r4, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r10, #8\n\t" - "lsr lr, lr, #24\n\t" + "lsl r2, r8, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "uxtb lr, r10, ror #16\n\t" + "uxtb r2, r8\n\t" #endif #else - "ubfx lr, r10, #16, #8\n\t" + "ubfx r2, r8, #0, #8\n\t" #endif - "eor r4, r4, r2, lsl #24\n\t" - "lsr r2, r9, #24\n\t" - "ldrb r5, [r0, r5, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" + "ldr r5, [r0, r5, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r9, #24\n\t" + "lsl r6, r11, #8\n\t" "lsr r6, r6, #24\n\t" #else - "uxtb r6, r9\n\t" + "uxtb r6, r11, ror #16\n\t" #endif #else - "ubfx r6, r9, #0, #8\n\t" + "ubfx r6, r11, #16, #8\n\t" #endif - "eor r5, r5, r7, lsl #8\n\t" + "eor r5, r5, r7, ror #24\n\t" + "lsr r7, r10, #24\n\t" + "eor r5, r5, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r8, #16\n\t" - "lsr r7, r7, #24\n\t" + "lsl lr, r8, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r7, r8, ror #8\n\t" + "uxtb lr, r8, ror #8\n\t" #endif #else - "ubfx r7, r8, #8, #8\n\t" + "ubfx lr, r8, #8, #8\n\t" #endif - "eor r5, r5, lr, lsl #16\n\t" + "eor r5, r5, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r11, #8\n\t" - "lsr lr, lr, #24\n\t" + "lsl r2, r9, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "uxtb lr, r11, ror #16\n\t" + "uxtb r2, r9\n\t" #endif #else - "ubfx lr, r11, #16, #8\n\t" + "ubfx r2, r9, #0, #8\n\t" #endif - "eor r5, r5, r2, lsl #24\n\t" - "lsr r2, r10, #24\n\t" - "ldrb r6, [r0, r6, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" - "lsr r11, r11, #24\n\t" - "eor r6, r6, r7, lsl #8\n\t" + "ldr r6, [r0, r6, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r10, #24\n\t" - "lsr r7, r7, #24\n\t" + "lsl r10, r10, #24\n\t" + "lsr r10, r10, #24\n\t" #else - "uxtb r7, r10\n\t" + "uxtb r10, r10\n\t" #endif #else - "ubfx r7, r10, #0, #8\n\t" + "ubfx r10, r10, #0, #8\n\t" #endif - "eor r6, r6, lr, lsl #16\n\t" + "eor r6, r6, r7, ror #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r9, #16\n\t" - "lsr lr, lr, #24\n\t" + "lsl r7, r8, #8\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb lr, r9, ror #8\n\t" + "uxtb r7, r8, ror #16\n\t" #endif #else - "ubfx lr, r9, #8, #8\n\t" + "ubfx r7, r8, #16, #8\n\t" #endif - "eor r6, r6, r2, lsl #24\n\t" + "eor r6, r6, lr, ror #8\n\t" + "lsr lr, r11, #24\n\t" + "eor r6, r6, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r8, #8\n\t" + "lsl r2, r9, #16\n\t" "lsr r2, r2, #24\n\t" #else - "uxtb r2, r8, ror #16\n\t" + "uxtb r2, r9, ror #8\n\t" #endif #else - "ubfx r2, r8, #16, #8\n\t" + "ubfx r2, r9, #8, #8\n\t" #endif - "ldrb r11, [r0, r11, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r11, lsl #16\n\t" - "ldm %[ks], {r8, r9, r10, r11}\n\t" - "eor r7, r7, lr, lsl #8\n\t" - "eor r7, r7, r2, lsl #16\n\t" - /* XOR in Key Schedule */ - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" -#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - "pop {r1, %[len], lr}\n\t" - "ldr %[ks], [sp]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "eor r8, r4, r4, ror #16\n\t" - "eor r9, r5, r5, ror #16\n\t" - "eor r10, r6, r6, ror #16\n\t" - "eor r11, r7, r7, ror #16\n\t" - "bic r8, r8, #0xff0000\n\t" - "bic r9, r9, #0xff0000\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "ror r5, r5, #8\n\t" - "ror r6, r6, #8\n\t" - "ror r7, r7, #8\n\t" - "eor r4, r4, r8, lsr #8\n\t" - "eor r5, r5, r9, lsr #8\n\t" - "eor r6, r6, r10, lsr #8\n\t" - "eor r7, r7, r11, lsr #8\n\t" -#else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "ldr r8, [lr]\n\t" - "ldr r9, [lr, #4]\n\t" - "ldr r10, [lr, #8]\n\t" - "ldr r11, [lr, #12]\n\t" - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" - "ldr r8, [sp, #4]\n\t" - "str r4, [%[out]]\n\t" - "str r5, [%[out], #4]\n\t" - "str r6, [%[out], #8]\n\t" - "str r7, [%[out], #12]\n\t" - "ldm r8, {r4, r5, r6, r7}\n\t" - "subs %[len], %[len], #16\n\t" - "add lr, lr, #16\n\t" - "add %[out], %[out], #16\n\t" - "bne L_AES_GCM_encrypt_loop_block_256_%=\n\t" - "b L_AES_GCM_encrypt_end_%=\n\t" - "\n" - "L_AES_GCM_encrypt_start_block_192_%=:\n\t" - "\n" - "L_AES_GCM_encrypt_loop_block_192_%=:\n\t" - "push {r1, %[len], lr}\n\t" - "ldr lr, [sp, #16]\n\t" - "add r7, r7, #1\n\t" + "ldr r10, [r0, r10, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r10, ror #24\n\t" "ldm %[ks]!, {r8, r9, r10, r11}\n\t" - "str r7, [lr, #12]\n\t" - /* Round: 0 - XOR in key schedule */ + "eor r7, r7, lr, ror #24\n\t" + "eor r7, r7, r2, ror #8\n\t" + /* XOR in Key Schedule */ "eor r4, r4, r8\n\t" "eor r5, r5, r9\n\t" "eor r6, r6, r10\n\t" "eor r7, r7, r11\n\t" - "mov r1, #5\n\t" -#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE - "bl AES_encrypt_block\n\t" -#else - "\n" - "L_AES_GCM_encrypt_block_nr_192_%=:\n\t" + "subs r1, r1, #1\n\t" + "bne L_AES_GCM_encrypt_block_nr_192_%=\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) "lsl r8, r5, #8\n\t" @@ -26122,164 +17653,227 @@ WC_OMIT_FRAME_POINTER void AES_GCM_encrypt(const unsigned char* in, "eor r11, r11, r7\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r9, #8\n\t" + "lsl r4, r11, #24\n\t" "lsr r4, r4, #24\n\t" #else - "uxtb r4, r9, ror #16\n\t" + "uxtb r4, r11\n\t" #endif #else - "ubfx r4, r9, #16, #8\n\t" + "ubfx r4, r11, #0, #8\n\t" #endif - "lsr r7, r8, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r10, #16\n\t" - "lsr lr, lr, #24\n\t" + "lsl r7, r10, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb lr, r10, ror #8\n\t" + "uxtb r7, r10, ror #8\n\t" #endif #else - "ubfx lr, r10, #8, #8\n\t" + "ubfx r7, r10, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r11, #24\n\t" - "lsr r2, r2, #24\n\t" + "lsl lr, r9, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r2, r11\n\t" + "uxtb lr, r9, ror #16\n\t" #endif #else - "ubfx r2, r11, #0, #8\n\t" + "ubfx lr, r9, #16, #8\n\t" #endif - "ldr r4, [r0, r4, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "lsr r2, r8, #24\n\t" + "ldrb r4, [r0, r4, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r10, #8\n\t" + "lsl r5, r8, #24\n\t" "lsr r5, r5, #24\n\t" #else - "uxtb r5, r10, ror #16\n\t" + "uxtb r5, r8\n\t" #endif #else - "ubfx r5, r10, #16, #8\n\t" + "ubfx r5, r8, #0, #8\n\t" #endif - "eor r4, r4, r7, ror #24\n\t" - "lsr r7, r9, #24\n\t" - "eor r4, r4, lr, ror #8\n\t" + "eor r4, r4, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r11, #16\n\t" - "lsr lr, lr, #24\n\t" + "lsl r7, r11, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb lr, r11, ror #8\n\t" + "uxtb r7, r11, ror #8\n\t" #endif #else - "ubfx lr, r11, #8, #8\n\t" + "ubfx r7, r11, #8, #8\n\t" #endif - "eor r4, r4, r2, ror #16\n\t" + "eor r4, r4, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r8, #24\n\t" - "lsr r2, r2, #24\n\t" + "lsl lr, r10, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r2, r8\n\t" + "uxtb lr, r10, ror #16\n\t" #endif #else - "ubfx r2, r8, #0, #8\n\t" + "ubfx lr, r10, #16, #8\n\t" #endif - "ldr r5, [r0, r5, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "eor r4, r4, r2, lsl #24\n\t" + "lsr r2, r9, #24\n\t" + "ldrb r5, [r0, r5, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r11, #8\n\t" + "lsl r6, r9, #24\n\t" "lsr r6, r6, #24\n\t" #else - "uxtb r6, r11, ror #16\n\t" + "uxtb r6, r9\n\t" #endif #else - "ubfx r6, r11, #16, #8\n\t" + "ubfx r6, r9, #0, #8\n\t" #endif - "eor r5, r5, r7, ror #24\n\t" - "lsr r7, r10, #24\n\t" - "eor r5, r5, lr, ror #8\n\t" + "eor r5, r5, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r8, #16\n\t" - "lsr lr, lr, #24\n\t" + "lsl r7, r8, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb lr, r8, ror #8\n\t" + "uxtb r7, r8, ror #8\n\t" #endif #else - "ubfx lr, r8, #8, #8\n\t" + "ubfx r7, r8, #8, #8\n\t" #endif - "eor r5, r5, r2, ror #16\n\t" + "eor r5, r5, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r9, #24\n\t" - "lsr r2, r2, #24\n\t" + "lsl lr, r11, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r2, r9\n\t" + "uxtb lr, r11, ror #16\n\t" #endif #else - "ubfx r2, r9, #0, #8\n\t" + "ubfx lr, r11, #16, #8\n\t" #endif - "ldr r6, [r0, r6, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "eor r5, r5, r2, lsl #24\n\t" + "lsr r2, r10, #24\n\t" + "ldrb r6, [r0, r6, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" + "lsr r11, r11, #24\n\t" + "eor r6, r6, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r10, r10, #24\n\t" - "lsr r10, r10, #24\n\t" + "lsl r7, r10, #24\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb r10, r10\n\t" + "uxtb r7, r10\n\t" #endif #else - "ubfx r10, r10, #0, #8\n\t" + "ubfx r7, r10, #0, #8\n\t" #endif - "eor r6, r6, r7, ror #24\n\t" + "eor r6, r6, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r8, #8\n\t" - "lsr r7, r7, #24\n\t" + "lsl lr, r9, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r7, r8, ror #16\n\t" + "uxtb lr, r9, ror #8\n\t" #endif #else - "ubfx r7, r8, #16, #8\n\t" + "ubfx lr, r9, #8, #8\n\t" #endif - "eor r6, r6, lr, ror #8\n\t" - "lsr lr, r11, #24\n\t" - "eor r6, r6, r2, ror #16\n\t" + "eor r6, r6, r2, lsl #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r9, #16\n\t" + "lsl r2, r8, #8\n\t" "lsr r2, r2, #24\n\t" #else - "uxtb r2, r9, ror #8\n\t" + "uxtb r2, r8, ror #16\n\t" #endif #else - "ubfx r2, r9, #8, #8\n\t" + "ubfx r2, r8, #16, #8\n\t" #endif - "ldr r10, [r0, r10, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r10, ror #24\n\t" - "ldm %[ks]!, {r8, r9, r10, r11}\n\t" - "eor r7, r7, lr, ror #24\n\t" - "eor r7, r7, r2, ror #8\n\t" + "ldrb r11, [r0, r11, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r11, lsl #16\n\t" + "ldm %[ks], {r8, r9, r10, r11}\n\t" + "eor r7, r7, lr, lsl #8\n\t" + "eor r7, r7, r2, lsl #16\n\t" /* XOR in Key Schedule */ "eor r4, r4, r8\n\t" "eor r5, r5, r9\n\t" "eor r6, r6, r10\n\t" "eor r7, r7, r11\n\t" - "subs r1, r1, #1\n\t" - "bne L_AES_GCM_encrypt_block_nr_192_%=\n\t" +#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ + "pop {r1, %[len], lr}\n\t" + "ldr %[ks], [sp]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "eor r8, r4, r4, ror #16\n\t" + "eor r9, r5, r5, ror #16\n\t" + "eor r10, r6, r6, ror #16\n\t" + "eor r11, r7, r7, ror #16\n\t" + "bic r8, r8, #0xff0000\n\t" + "bic r9, r9, #0xff0000\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "ror r5, r5, #8\n\t" + "ror r6, r6, #8\n\t" + "ror r7, r7, #8\n\t" + "eor r4, r4, r8, lsr #8\n\t" + "eor r5, r5, r9, lsr #8\n\t" + "eor r6, r6, r10, lsr #8\n\t" + "eor r7, r7, r11, lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "ldr r8, [lr]\n\t" + "ldr r9, [lr, #4]\n\t" + "ldr r10, [lr, #8]\n\t" + "ldr r11, [lr, #12]\n\t" + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "ldr r8, [sp, #4]\n\t" + "str r4, [%[out]]\n\t" + "str r5, [%[out], #4]\n\t" + "str r6, [%[out], #8]\n\t" + "str r7, [%[out], #12]\n\t" + "ldm r8, {r4, r5, r6, r7}\n\t" + "subs %[len], %[len], #16\n\t" + "add lr, lr, #16\n\t" + "add %[out], %[out], #16\n\t" + "bne L_AES_GCM_encrypt_loop_block_192_%=\n\t" + "b L_AES_GCM_encrypt_end_%=\n\t" + "\n" + "L_AES_GCM_encrypt_start_block_128_%=:\n\t" + "\n" + "L_AES_GCM_encrypt_loop_block_128_%=:\n\t" + "push {r1, %[len], lr}\n\t" + "ldr lr, [sp, #16]\n\t" + "add r7, r7, #1\n\t" + "ldm %[ks]!, {r8, r9, r10, r11}\n\t" + "str r7, [lr, #12]\n\t" + /* Round: 0 - XOR in key schedule */ + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "mov r1, #4\n\t" +#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE + "bl AES_encrypt_block\n\t" +#else + "\n" + "L_AES_GCM_encrypt_block_nr_128_%=:\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) "lsl r8, r5, #8\n\t" @@ -26440,227 +18034,164 @@ WC_OMIT_FRAME_POINTER void AES_GCM_encrypt(const unsigned char* in, "eor r11, r11, r7\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r11, #24\n\t" + "lsl r4, r9, #8\n\t" "lsr r4, r4, #24\n\t" #else - "uxtb r4, r11\n\t" + "uxtb r4, r9, ror #16\n\t" #endif #else - "ubfx r4, r11, #0, #8\n\t" + "ubfx r4, r9, #16, #8\n\t" #endif + "lsr r7, r8, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r10, #16\n\t" - "lsr r7, r7, #24\n\t" + "lsl lr, r10, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r7, r10, ror #8\n\t" + "uxtb lr, r10, ror #8\n\t" #endif #else - "ubfx r7, r10, #8, #8\n\t" + "ubfx lr, r10, #8, #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r9, #8\n\t" - "lsr lr, lr, #24\n\t" + "lsl r2, r11, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "uxtb lr, r9, ror #16\n\t" + "uxtb r2, r11\n\t" #endif #else - "ubfx lr, r9, #16, #8\n\t" + "ubfx r2, r11, #0, #8\n\t" #endif - "lsr r2, r8, #24\n\t" - "ldrb r4, [r0, r4, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" + "ldr r4, [r0, r4, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r8, #24\n\t" + "lsl r5, r10, #8\n\t" "lsr r5, r5, #24\n\t" #else - "uxtb r5, r8\n\t" + "uxtb r5, r10, ror #16\n\t" #endif #else - "ubfx r5, r8, #0, #8\n\t" + "ubfx r5, r10, #16, #8\n\t" #endif - "eor r4, r4, r7, lsl #8\n\t" + "eor r4, r4, r7, ror #24\n\t" + "lsr r7, r9, #24\n\t" + "eor r4, r4, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r11, #16\n\t" - "lsr r7, r7, #24\n\t" + "lsl lr, r11, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r7, r11, ror #8\n\t" + "uxtb lr, r11, ror #8\n\t" #endif #else - "ubfx r7, r11, #8, #8\n\t" + "ubfx lr, r11, #8, #8\n\t" #endif - "eor r4, r4, lr, lsl #16\n\t" + "eor r4, r4, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r10, #8\n\t" - "lsr lr, lr, #24\n\t" + "lsl r2, r8, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "uxtb lr, r10, ror #16\n\t" + "uxtb r2, r8\n\t" #endif #else - "ubfx lr, r10, #16, #8\n\t" + "ubfx r2, r8, #0, #8\n\t" #endif - "eor r4, r4, r2, lsl #24\n\t" - "lsr r2, r9, #24\n\t" - "ldrb r5, [r0, r5, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" + "ldr r5, [r0, r5, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r9, #24\n\t" + "lsl r6, r11, #8\n\t" "lsr r6, r6, #24\n\t" #else - "uxtb r6, r9\n\t" + "uxtb r6, r11, ror #16\n\t" #endif #else - "ubfx r6, r9, #0, #8\n\t" + "ubfx r6, r11, #16, #8\n\t" #endif - "eor r5, r5, r7, lsl #8\n\t" + "eor r5, r5, r7, ror #24\n\t" + "lsr r7, r10, #24\n\t" + "eor r5, r5, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r8, #16\n\t" - "lsr r7, r7, #24\n\t" + "lsl lr, r8, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r7, r8, ror #8\n\t" + "uxtb lr, r8, ror #8\n\t" #endif #else - "ubfx r7, r8, #8, #8\n\t" + "ubfx lr, r8, #8, #8\n\t" #endif - "eor r5, r5, lr, lsl #16\n\t" + "eor r5, r5, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r11, #8\n\t" - "lsr lr, lr, #24\n\t" + "lsl r2, r9, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "uxtb lr, r11, ror #16\n\t" + "uxtb r2, r9\n\t" #endif #else - "ubfx lr, r11, #16, #8\n\t" + "ubfx r2, r9, #0, #8\n\t" #endif - "eor r5, r5, r2, lsl #24\n\t" - "lsr r2, r10, #24\n\t" - "ldrb r6, [r0, r6, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" - "lsr r11, r11, #24\n\t" - "eor r6, r6, r7, lsl #8\n\t" + "ldr r6, [r0, r6, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r10, #24\n\t" - "lsr r7, r7, #24\n\t" + "lsl r10, r10, #24\n\t" + "lsr r10, r10, #24\n\t" #else - "uxtb r7, r10\n\t" + "uxtb r10, r10\n\t" #endif #else - "ubfx r7, r10, #0, #8\n\t" + "ubfx r10, r10, #0, #8\n\t" #endif - "eor r6, r6, lr, lsl #16\n\t" + "eor r6, r6, r7, ror #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r9, #16\n\t" - "lsr lr, lr, #24\n\t" + "lsl r7, r8, #8\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb lr, r9, ror #8\n\t" + "uxtb r7, r8, ror #16\n\t" #endif #else - "ubfx lr, r9, #8, #8\n\t" + "ubfx r7, r8, #16, #8\n\t" #endif - "eor r6, r6, r2, lsl #24\n\t" + "eor r6, r6, lr, ror #8\n\t" + "lsr lr, r11, #24\n\t" + "eor r6, r6, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r8, #8\n\t" + "lsl r2, r9, #16\n\t" "lsr r2, r2, #24\n\t" #else - "uxtb r2, r8, ror #16\n\t" + "uxtb r2, r9, ror #8\n\t" #endif #else - "ubfx r2, r8, #16, #8\n\t" + "ubfx r2, r9, #8, #8\n\t" #endif - "ldrb r11, [r0, r11, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r11, lsl #16\n\t" - "ldm %[ks], {r8, r9, r10, r11}\n\t" - "eor r7, r7, lr, lsl #8\n\t" - "eor r7, r7, r2, lsl #16\n\t" - /* XOR in Key Schedule */ - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" -#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - "pop {r1, %[len], lr}\n\t" - "ldr %[ks], [sp]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "eor r8, r4, r4, ror #16\n\t" - "eor r9, r5, r5, ror #16\n\t" - "eor r10, r6, r6, ror #16\n\t" - "eor r11, r7, r7, ror #16\n\t" - "bic r8, r8, #0xff0000\n\t" - "bic r9, r9, #0xff0000\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "ror r5, r5, #8\n\t" - "ror r6, r6, #8\n\t" - "ror r7, r7, #8\n\t" - "eor r4, r4, r8, lsr #8\n\t" - "eor r5, r5, r9, lsr #8\n\t" - "eor r6, r6, r10, lsr #8\n\t" - "eor r7, r7, r11, lsr #8\n\t" -#else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "ldr r8, [lr]\n\t" - "ldr r9, [lr, #4]\n\t" - "ldr r10, [lr, #8]\n\t" - "ldr r11, [lr, #12]\n\t" - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" - "ldr r8, [sp, #4]\n\t" - "str r4, [%[out]]\n\t" - "str r5, [%[out], #4]\n\t" - "str r6, [%[out], #8]\n\t" - "str r7, [%[out], #12]\n\t" - "ldm r8, {r4, r5, r6, r7}\n\t" - "subs %[len], %[len], #16\n\t" - "add lr, lr, #16\n\t" - "add %[out], %[out], #16\n\t" - "bne L_AES_GCM_encrypt_loop_block_192_%=\n\t" - "b L_AES_GCM_encrypt_end_%=\n\t" - "\n" - "L_AES_GCM_encrypt_start_block_128_%=:\n\t" - "\n" - "L_AES_GCM_encrypt_loop_block_128_%=:\n\t" - "push {r1, %[len], lr}\n\t" - "ldr lr, [sp, #16]\n\t" - "add r7, r7, #1\n\t" + "ldr r10, [r0, r10, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r10, ror #24\n\t" "ldm %[ks]!, {r8, r9, r10, r11}\n\t" - "str r7, [lr, #12]\n\t" - /* Round: 0 - XOR in key schedule */ + "eor r7, r7, lr, ror #24\n\t" + "eor r7, r7, r2, ror #8\n\t" + /* XOR in Key Schedule */ "eor r4, r4, r8\n\t" "eor r5, r5, r9\n\t" "eor r6, r6, r10\n\t" "eor r7, r7, r11\n\t" - "mov r1, #4\n\t" -#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE - "bl AES_encrypt_block\n\t" -#else - "\n" - "L_AES_GCM_encrypt_block_nr_128_%=:\n\t" + "subs r1, r1, #1\n\t" + "bne L_AES_GCM_encrypt_block_nr_128_%=\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) "lsl r8, r5, #8\n\t" @@ -26760,1631 +18291,2002 @@ WC_OMIT_FRAME_POINTER void AES_GCM_encrypt(const unsigned char* in, "eor r9, r9, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #24\n\t" - "lsr r2, r2, #24\n\t" + "lsl r2, r5, #24\n\t" + "lsr r2, r2, #24\n\t" +#else + "uxtb r2, r5\n\t" +#endif +#else + "ubfx r2, r5, #0, #8\n\t" +#endif + "ldr r10, [r0, r10, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r6, #24\n\t" + "lsr r6, r6, #24\n\t" +#else + "uxtb r6, r6\n\t" +#endif +#else + "ubfx r6, r6, #0, #8\n\t" +#endif + "eor r10, r10, r11, ror #24\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r11, r4, #8\n\t" + "lsr r11, r11, #24\n\t" +#else + "uxtb r11, r4, ror #16\n\t" +#endif +#else + "ubfx r11, r4, #16, #8\n\t" +#endif + "eor r10, r10, lr, ror #8\n\t" + "lsr lr, r7, #24\n\t" + "eor r10, r10, r2, ror #16\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r5, #16\n\t" + "lsr r2, r2, #24\n\t" +#else + "uxtb r2, r5, ror #8\n\t" +#endif +#else + "ubfx r2, r5, #8, #8\n\t" +#endif + "ldr r6, [r0, r6, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r6, ror #24\n\t" + "ldm %[ks]!, {r4, r5, r6, r7}\n\t" + "eor r11, r11, lr, ror #24\n\t" + "eor r11, r11, r2, ror #8\n\t" + /* XOR in Key Schedule */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r11, #24\n\t" + "lsr r4, r4, #24\n\t" +#else + "uxtb r4, r11\n\t" +#endif +#else + "ubfx r4, r11, #0, #8\n\t" +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r10, #16\n\t" + "lsr r7, r7, #24\n\t" +#else + "uxtb r7, r10, ror #8\n\t" +#endif +#else + "ubfx r7, r10, #8, #8\n\t" +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r9, #8\n\t" + "lsr lr, lr, #24\n\t" +#else + "uxtb lr, r9, ror #16\n\t" +#endif +#else + "ubfx lr, r9, #16, #8\n\t" +#endif + "lsr r2, r8, #24\n\t" + "ldrb r4, [r0, r4, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r5, r8, #24\n\t" + "lsr r5, r5, #24\n\t" +#else + "uxtb r5, r8\n\t" +#endif +#else + "ubfx r5, r8, #0, #8\n\t" +#endif + "eor r4, r4, r7, lsl #8\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r11, #16\n\t" + "lsr r7, r7, #24\n\t" +#else + "uxtb r7, r11, ror #8\n\t" +#endif +#else + "ubfx r7, r11, #8, #8\n\t" +#endif + "eor r4, r4, lr, lsl #16\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r10, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r2, r5\n\t" + "uxtb lr, r10, ror #16\n\t" #endif #else - "ubfx r2, r5, #0, #8\n\t" + "ubfx lr, r10, #16, #8\n\t" #endif - "ldr r10, [r0, r10, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "eor r4, r4, r2, lsl #24\n\t" + "lsr r2, r9, #24\n\t" + "ldrb r5, [r0, r5, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r6, #24\n\t" + "lsl r6, r9, #24\n\t" "lsr r6, r6, #24\n\t" #else - "uxtb r6, r6\n\t" + "uxtb r6, r9\n\t" #endif #else - "ubfx r6, r6, #0, #8\n\t" + "ubfx r6, r9, #0, #8\n\t" #endif - "eor r10, r10, r11, ror #24\n\t" + "eor r5, r5, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r11, r4, #8\n\t" - "lsr r11, r11, #24\n\t" + "lsl r7, r8, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb r11, r4, ror #16\n\t" + "uxtb r7, r8, ror #8\n\t" #endif #else - "ubfx r11, r4, #16, #8\n\t" + "ubfx r7, r8, #8, #8\n\t" #endif - "eor r10, r10, lr, ror #8\n\t" - "lsr lr, r7, #24\n\t" - "eor r10, r10, r2, ror #16\n\t" + "eor r5, r5, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #16\n\t" - "lsr r2, r2, #24\n\t" + "lsl lr, r11, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r2, r5, ror #8\n\t" + "uxtb lr, r11, ror #16\n\t" #endif #else - "ubfx r2, r5, #8, #8\n\t" + "ubfx lr, r11, #16, #8\n\t" #endif - "ldr r6, [r0, r6, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r6, ror #24\n\t" - "ldm %[ks]!, {r4, r5, r6, r7}\n\t" - "eor r11, r11, lr, ror #24\n\t" - "eor r11, r11, r2, ror #8\n\t" - /* XOR in Key Schedule */ - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" + "eor r5, r5, r2, lsl #24\n\t" + "lsr r2, r10, #24\n\t" + "ldrb r6, [r0, r6, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" + "lsr r11, r11, #24\n\t" + "eor r6, r6, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r9, #8\n\t" - "lsr r4, r4, #24\n\t" + "lsl r7, r10, #24\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb r4, r9, ror #16\n\t" + "uxtb r7, r10\n\t" #endif #else - "ubfx r4, r9, #16, #8\n\t" + "ubfx r7, r10, #0, #8\n\t" #endif - "lsr r7, r8, #24\n\t" + "eor r6, r6, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r10, #16\n\t" + "lsl lr, r9, #16\n\t" "lsr lr, lr, #24\n\t" #else - "uxtb lr, r10, ror #8\n\t" + "uxtb lr, r9, ror #8\n\t" #endif #else - "ubfx lr, r10, #8, #8\n\t" + "ubfx lr, r9, #8, #8\n\t" #endif + "eor r6, r6, r2, lsl #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r11, #24\n\t" + "lsl r2, r8, #8\n\t" "lsr r2, r2, #24\n\t" #else - "uxtb r2, r11\n\t" + "uxtb r2, r8, ror #16\n\t" #endif #else - "ubfx r2, r11, #0, #8\n\t" + "ubfx r2, r8, #16, #8\n\t" #endif - "ldr r4, [r0, r4, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "ldrb r11, [r0, r11, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r11, lsl #16\n\t" + "ldm %[ks], {r8, r9, r10, r11}\n\t" + "eor r7, r7, lr, lsl #8\n\t" + "eor r7, r7, r2, lsl #16\n\t" + /* XOR in Key Schedule */ + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" +#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ + "pop {r1, %[len], lr}\n\t" + "ldr %[ks], [sp]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "eor r8, r4, r4, ror #16\n\t" + "eor r9, r5, r5, ror #16\n\t" + "eor r10, r6, r6, ror #16\n\t" + "eor r11, r7, r7, ror #16\n\t" + "bic r8, r8, #0xff0000\n\t" + "bic r9, r9, #0xff0000\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "ror r5, r5, #8\n\t" + "ror r6, r6, #8\n\t" + "ror r7, r7, #8\n\t" + "eor r4, r4, r8, lsr #8\n\t" + "eor r5, r5, r9, lsr #8\n\t" + "eor r6, r6, r10, lsr #8\n\t" + "eor r7, r7, r11, lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "ldr r8, [lr]\n\t" + "ldr r9, [lr, #4]\n\t" + "ldr r10, [lr, #8]\n\t" + "ldr r11, [lr, #12]\n\t" + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "ldr r8, [sp, #4]\n\t" + "str r4, [%[out]]\n\t" + "str r5, [%[out], #4]\n\t" + "str r6, [%[out], #8]\n\t" + "str r7, [%[out], #12]\n\t" + "ldm r8, {r4, r5, r6, r7}\n\t" + "subs %[len], %[len], #16\n\t" + "add lr, lr, #16\n\t" + "add %[out], %[out], #16\n\t" + "bne L_AES_GCM_encrypt_loop_block_128_%=\n\t" + "\n" + "L_AES_GCM_encrypt_end_%=:\n\t" + "pop {%[ks], r8}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "eor r10, r4, r4, ror #16\n\t" + "eor r11, r5, r5, ror #16\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "ror r5, r5, #8\n\t" + "eor r4, r4, r10, lsr #8\n\t" + "eor r5, r5, r11, lsr #8\n\t" + "eor r10, r6, r6, ror #16\n\t" + "eor r11, r7, r7, ror #16\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "ror r7, r7, #8\n\t" + "eor r6, r6, r10, lsr #8\n\t" + "eor r7, r7, r11, lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "stm r8, {r4, r5, r6, r7}\n\t" + "pop {%[nr], %[ctr]}\n\t" + "pop {%[L_AES_ARM32_te_gcm]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [len] "+r" (len), [ks] "+r" (ks), + [nr] "+r" (nr), [ctr] "+r" (ctr), + [L_AES_ARM32_te_gcm] "+r" (L_AES_ARM32_te_gcm_c) + : +#else + : + : [in] "r" (in), [out] "r" (out), [len] "r" (len), [ks] "r" (ks), + [nr] "r" (nr), [ctr] "r" (ctr), + [L_AES_ARM32_te_gcm] "r" (L_AES_ARM32_te_gcm_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r5", "r6", "r7", "r8", "r9", "r10", "r11" + ); +} + +#endif /* HAVE_AESGCM */ +#ifdef WOLFSSL_AESGCM_SIV +XALIGNED(8) static const word32 L_AES_GCMSIV_polyval_base_r[] = { + 0x00000000, 0x1c200000, 0x38400000, 0x24600000, + 0x70800000, 0x6ca00000, 0x48c00000, 0x54e00000, + 0xe1000000, 0xfd200000, 0xd9400000, 0xc5600000, + 0x91800000, 0x8da00000, 0xa9c00000, 0xb5e00000, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_GCMSIV_polyval_base(unsigned char* s_p, + const unsigned char* m_p, const unsigned char* data_p, + unsigned int blocks_p) +#else +WC_OMIT_FRAME_POINTER void AES_GCMSIV_polyval_base(unsigned char* s, + const unsigned char* m, const unsigned char* data, unsigned int blocks) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register unsigned char* s __asm__ ("r0") = (unsigned char*)s_p; + register const unsigned char* m __asm__ ("r1") = (const unsigned char*)m_p; + register const unsigned char* data __asm__ ("r2") = + (const unsigned char*)data_p; + register unsigned int blocks __asm__ ("r3") = (unsigned int)blocks_p; + register word32* L_AES_ARM32_te_gcm_c __asm__ ("r12") = + (word32*)L_AES_ARM32_te_gcm; + register word32* L_AES_GCMSIV_polyval_base_r_c __asm__ ("lr") = + (word32*)&L_AES_GCMSIV_polyval_base_r; +#else + register word32* L_AES_ARM32_te_gcm_c = (word32*)L_AES_ARM32_te_gcm; + register word32* L_AES_GCMSIV_polyval_base_r_c = + (word32*)&L_AES_GCMSIV_polyval_base_r; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_ARM32_te_gcm], %[L_AES_GCMSIV_polyval_base_r]}\n\t" + "mov r6, %[L_AES_GCMSIV_polyval_base_r]\n\t" + "cmp %[blocks], #0\n\t" + "beq L_AES_GCMSIV_polyval_base_done_%=\n\t" + "\n" + "L_AES_GCMSIV_polyval_base_loop_%=:\n\t" + "ldr r10, [%[data], #12]\n\t" + "rev r10, r10\n\t" + "ldr r8, [%[s]]\n\t" + "eor r8, r8, r10\n\t" + "str r8, [%[s]]\n\t" + "ldr r10, [%[data], #8]\n\t" + "rev r10, r10\n\t" + "ldr r8, [%[s], #4]\n\t" + "eor r8, r8, r10\n\t" + "str r8, [%[s], #4]\n\t" + "ldr r10, [%[data], #4]\n\t" + "rev r10, r10\n\t" + "ldr r8, [%[s], #8]\n\t" + "eor r8, r8, r10\n\t" + "str r8, [%[s], #8]\n\t" + "ldr r10, [%[data]]\n\t" + "rev r10, r10\n\t" + "ldr r8, [%[s], #12]\n\t" + "eor r8, r8, r10\n\t" + "str r8, [%[s], #12]\n\t" + "mov r12, #0\n\t" + "mov lr, #0\n\t" + "mov r4, #0\n\t" + "mov r5, #0\n\t" + "ldr r7, [%[s], #12]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r10, #8\n\t" - "lsr r5, r5, #24\n\t" -#else - "uxtb r5, r10, ror #16\n\t" -#endif + "lsl r8, r7, #4\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r5, r10, #16, #8\n\t" + "ubfx r8, r7, #24, #4\n\t" #endif - "eor r4, r4, r7, ror #24\n\t" - "lsr r7, r9, #24\n\t" - "eor r4, r4, lr, ror #8\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r11, #16\n\t" - "lsr lr, lr, #24\n\t" -#else - "uxtb lr, r11, ror #8\n\t" -#endif + "lsr r8, r7, #28\n\t" #else - "ubfx lr, r11, #8, #8\n\t" + "ubfx r8, r7, #28, #4\n\t" #endif - "eor r4, r4, r2, ror #16\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r8, #24\n\t" - "lsr r2, r2, #24\n\t" -#else - "uxtb r2, r8\n\t" -#endif + "lsl r8, r7, #12\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r2, r8, #0, #8\n\t" + "ubfx r8, r7, #16, #4\n\t" #endif - "ldr r5, [r0, r5, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r11, #8\n\t" - "lsr r6, r6, #24\n\t" -#else - "uxtb r6, r11, ror #16\n\t" -#endif + "lsl r8, r7, #8\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r6, r11, #16, #8\n\t" + "ubfx r8, r7, #20, #4\n\t" #endif - "eor r5, r5, r7, ror #24\n\t" - "lsr r7, r10, #24\n\t" - "eor r5, r5, lr, ror #8\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r8, #16\n\t" - "lsr lr, lr, #24\n\t" -#else - "uxtb lr, r8, ror #8\n\t" -#endif + "lsl r8, r7, #20\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx lr, r8, #8, #8\n\t" + "ubfx r8, r7, #8, #4\n\t" #endif - "eor r5, r5, r2, ror #16\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r9, #24\n\t" - "lsr r2, r2, #24\n\t" -#else - "uxtb r2, r9\n\t" -#endif + "lsl r8, r7, #16\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r2, r9, #0, #8\n\t" + "ubfx r8, r7, #12, #4\n\t" #endif - "ldr r6, [r0, r6, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r10, r10, #24\n\t" - "lsr r10, r10, #24\n\t" -#else - "uxtb r10, r10\n\t" -#endif + "lsl r8, r7, #28\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r10, r10, #0, #8\n\t" + "ubfx r8, r7, #0, #4\n\t" #endif - "eor r6, r6, r7, ror #24\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r8, #8\n\t" - "lsr r7, r7, #24\n\t" -#else - "uxtb r7, r8, ror #16\n\t" -#endif + "lsl r8, r7, #24\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r7, r8, #16, #8\n\t" + "ubfx r8, r7, #4, #4\n\t" #endif - "eor r6, r6, lr, ror #8\n\t" - "lsr lr, r11, #24\n\t" - "eor r6, r6, r2, ror #16\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" + "ldr r7, [%[s], #8]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r9, #16\n\t" - "lsr r2, r2, #24\n\t" -#else - "uxtb r2, r9, ror #8\n\t" -#endif + "lsl r8, r7, #4\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r2, r9, #8, #8\n\t" + "ubfx r8, r7, #24, #4\n\t" #endif - "ldr r10, [r0, r10, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r10, ror #24\n\t" - "ldm %[ks]!, {r8, r9, r10, r11}\n\t" - "eor r7, r7, lr, ror #24\n\t" - "eor r7, r7, r2, ror #8\n\t" - /* XOR in Key Schedule */ - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" - "subs r1, r1, #1\n\t" - "bne L_AES_GCM_encrypt_block_nr_128_%=\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r8, r5, #8\n\t" - "lsr r8, r8, #24\n\t" + "lsr r8, r7, #28\n\t" #else - "uxtb r8, r5, ror #16\n\t" + "ubfx r8, r7, #28, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r8, r7, #12\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r8, r5, #16, #8\n\t" + "ubfx r8, r7, #16, #4\n\t" #endif - "lsr r11, r4, #24\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r6, #16\n\t" - "lsr lr, lr, #24\n\t" + "lsl r8, r7, #8\n\t" + "lsr r8, r8, #28\n\t" #else - "uxtb lr, r6, ror #8\n\t" + "ubfx r8, r7, #20, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r8, r7, #20\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx lr, r6, #8, #8\n\t" + "ubfx r8, r7, #8, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r7, #24\n\t" - "lsr r2, r2, #24\n\t" + "lsl r8, r7, #16\n\t" + "lsr r8, r8, #28\n\t" #else - "uxtb r2, r7\n\t" + "ubfx r8, r7, #12, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r8, r7, #28\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r2, r7, #0, #8\n\t" + "ubfx r8, r7, #0, #4\n\t" #endif - "ldr r8, [r0, r8, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r9, r6, #8\n\t" - "lsr r9, r9, #24\n\t" + "lsl r8, r7, #24\n\t" + "lsr r8, r8, #28\n\t" #else - "uxtb r9, r6, ror #16\n\t" + "ubfx r8, r7, #4, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" + "ldr r7, [%[s], #4]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r8, r7, #4\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r9, r6, #16, #8\n\t" + "ubfx r8, r7, #24, #4\n\t" #endif - "eor r8, r8, r11, ror #24\n\t" - "lsr r11, r5, #24\n\t" - "eor r8, r8, lr, ror #8\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r7, #16\n\t" - "lsr lr, lr, #24\n\t" + "lsr r8, r7, #28\n\t" #else - "uxtb lr, r7, ror #8\n\t" + "ubfx r8, r7, #28, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r8, r7, #12\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx lr, r7, #8, #8\n\t" + "ubfx r8, r7, #16, #4\n\t" #endif - "eor r8, r8, r2, ror #16\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r4, #24\n\t" - "lsr r2, r2, #24\n\t" + "lsl r8, r7, #8\n\t" + "lsr r8, r8, #28\n\t" #else - "uxtb r2, r4\n\t" + "ubfx r8, r7, #20, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r8, r7, #20\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r2, r4, #0, #8\n\t" + "ubfx r8, r7, #8, #4\n\t" #endif - "ldr r9, [r0, r9, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r10, r7, #8\n\t" - "lsr r10, r10, #24\n\t" + "lsl r8, r7, #16\n\t" + "lsr r8, r8, #28\n\t" #else - "uxtb r10, r7, ror #16\n\t" + "ubfx r8, r7, #12, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r8, r7, #28\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r10, r7, #16, #8\n\t" + "ubfx r8, r7, #0, #4\n\t" #endif - "eor r9, r9, r11, ror #24\n\t" - "lsr r11, r6, #24\n\t" - "eor r9, r9, lr, ror #8\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r4, #16\n\t" - "lsr lr, lr, #24\n\t" + "lsl r8, r7, #24\n\t" + "lsr r8, r8, #28\n\t" #else - "uxtb lr, r4, ror #8\n\t" + "ubfx r8, r7, #4, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" + "ldr r7, [%[s]]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r8, r7, #4\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx lr, r4, #8, #8\n\t" + "ubfx r8, r7, #24, #4\n\t" #endif - "eor r9, r9, r2, ror #16\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #24\n\t" - "lsr r2, r2, #24\n\t" + "lsr r8, r7, #28\n\t" #else - "uxtb r2, r5\n\t" + "ubfx r8, r7, #28, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r8, r7, #12\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r2, r5, #0, #8\n\t" + "ubfx r8, r7, #16, #4\n\t" #endif - "ldr r10, [r0, r10, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r6, #24\n\t" - "lsr r6, r6, #24\n\t" + "lsl r8, r7, #8\n\t" + "lsr r8, r8, #28\n\t" #else - "uxtb r6, r6\n\t" + "ubfx r8, r7, #20, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r8, r7, #20\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r6, r6, #0, #8\n\t" + "ubfx r8, r7, #8, #4\n\t" #endif - "eor r10, r10, r11, ror #24\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r11, r4, #8\n\t" - "lsr r11, r11, #24\n\t" + "lsl r8, r7, #16\n\t" + "lsr r8, r8, #28\n\t" #else - "uxtb r11, r4, ror #16\n\t" + "ubfx r8, r7, #12, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "lsl r8, r7, #28\n\t" + "lsr r8, r8, #28\n\t" #else - "ubfx r11, r4, #16, #8\n\t" + "ubfx r8, r7, #0, #4\n\t" #endif - "eor r10, r10, lr, ror #8\n\t" - "lsr lr, r7, #24\n\t" - "eor r10, r10, r2, ror #16\n\t" + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "and r8, r4, #15\n\t" + "lsr r4, r4, #4\n\t" + "orr r4, r4, r5, lsl #28\n\t" + "lsr r5, r5, #4\n\t" + "orr r5, r5, r12, lsl #28\n\t" + "lsr r12, r12, #4\n\t" + "orr r12, r12, lr, lsl #28\n\t" + "lsr lr, lr, #4\n\t" + "ldr r10, [r6, r8, lsl #2]\n\t" + "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #16\n\t" - "lsr r2, r2, #24\n\t" + "lsl r8, r7, #24\n\t" + "lsr r8, r8, #28\n\t" #else - "uxtb r2, r5, ror #8\n\t" + "ubfx r8, r7, #4, #4\n\t" #endif + "add r9, %[m], r8, lsl #4\n\t" + "ldr r10, [r9]\n\t" + "eor r12, r12, r10\n\t" + "ldr r10, [r9, #4]\n\t" + "eor lr, lr, r10\n\t" + "ldr r10, [r9, #8]\n\t" + "eor r4, r4, r10\n\t" + "ldr r10, [r9, #12]\n\t" + "eor r5, r5, r10\n\t" + "rev lr, lr\n\t" + "rev r12, r12\n\t" + "rev r5, r5\n\t" + "rev r4, r4\n\t" + "str lr, [%[s]]\n\t" + "str r12, [%[s], #4]\n\t" + "str r5, [%[s], #8]\n\t" + "str r4, [%[s], #12]\n\t" + "subs %[blocks], %[blocks], #1\n\t" + "add %[data], %[data], #16\n\t" + "bne L_AES_GCMSIV_polyval_base_loop_%=\n\t" + "\n" + "L_AES_GCMSIV_polyval_base_done_%=:\n\t" + "pop {%[L_AES_ARM32_te_gcm], %[L_AES_GCMSIV_polyval_base_r]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [s] "+r" (s), [m] "+r" (m), [data] "+r" (data), + [blocks] "+r" (blocks), + [L_AES_ARM32_te_gcm] "+r" (L_AES_ARM32_te_gcm_c), + [L_AES_GCMSIV_polyval_base_r] "+r" (L_AES_GCMSIV_polyval_base_r_c) + : #else - "ubfx r2, r5, #8, #8\n\t" -#endif - "ldr r6, [r0, r6, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r6, ror #24\n\t" - "ldm %[ks]!, {r4, r5, r6, r7}\n\t" - "eor r11, r11, lr, ror #24\n\t" - "eor r11, r11, r2, ror #8\n\t" - /* XOR in Key Schedule */ - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + : + : [s] "r" (s), [m] "r" (m), [data] "r" (data), [blocks] "r" (blocks), + [L_AES_ARM32_te_gcm] "r" (L_AES_ARM32_te_gcm_c), + [L_AES_GCMSIV_polyval_base_r] "r" (L_AES_GCMSIV_polyval_base_r_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r4", "r5", "r6", "r7", "r8", "r9", "r10" + ); +} + +XALIGNED(8) static const word32 L_AES_GCMSIV_ctr_base_te_data[] = { + 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, + 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, + 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, + 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, + 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, + 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, + 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, + 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, + 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, + 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, + 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, + 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, + 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, + 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, + 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, + 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, + 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, + 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, + 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, + 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, + 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, + 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, + 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, + 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, + 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, + 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, + 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, + 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, + 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, + 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, + 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, + 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, + 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, + 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, + 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, + 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, + 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, + 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, + 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, + 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, + 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, + 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, + 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, + 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, + 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, + 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, + 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, + 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, + 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, + 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, + 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, + 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, + 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, + 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, + 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, + 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, + 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, + 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, + 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, + 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, + 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, + 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, + 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, + 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, +}; + +static const word32* L_AES_GCMSIV_ctr_base_te = L_AES_GCMSIV_ctr_base_te_data; +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in_p, + unsigned char* out_p, unsigned long length_p, const unsigned char* KS_p, + int nr_p, unsigned char* ctr_p) +#else +WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const unsigned char* in __asm__ ("r0") = + (const unsigned char*)in_p; + register unsigned char* out __asm__ ("r1") = (unsigned char*)out_p; + register unsigned long length __asm__ ("r2") = (unsigned long)length_p; + register const unsigned char* KS __asm__ ("r3") = + (const unsigned char*)KS_p; + register int nr __asm__ ("r12") = (int)nr_p; + register unsigned char* ctr __asm__ ("lr") = (unsigned char*)ctr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r4") = + (word32*)L_AES_GCMSIV_ctr_base_te; +#else + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_GCMSIV_ctr_base_te]}\n\t" + "push {%[nr], %[ctr]}\n\t" + "ldr r12, [sp]\n\t" + "ldr r8, [sp, #4]\n\t" + "mov lr, %[in]\n\t" + "ldr r0, [sp, #8]\n\t" + "ldm r8, {r4, r5, r6, r7}\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r11, #24\n\t" - "lsr r4, r4, #24\n\t" + "eor r10, r4, r4, ror #16\n\t" + "eor r11, r5, r5, ror #16\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "ror r5, r5, #8\n\t" + "eor r4, r4, r10, lsr #8\n\t" + "eor r5, r5, r11, lsr #8\n\t" + "eor r10, r6, r6, ror #16\n\t" + "eor r11, r7, r7, ror #16\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "ror r7, r7, #8\n\t" + "eor r6, r6, r10, lsr #8\n\t" + "eor r7, r7, r11, lsr #8\n\t" #else - "uxtb r4, r11\n\t" -#endif + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "stm r8, {r4, r5, r6, r7}\n\t" + "push {%[KS], r8}\n\t" + "cmp r12, #10\n\t" + "beq L_AES_GCMSIV_ctr_base_start_block_128_%=\n\t" + "cmp r12, #12\n\t" + "beq L_AES_GCMSIV_ctr_base_start_block_192_%=\n\t" + "\n" + "L_AES_GCMSIV_ctr_base_loop_block_256_%=:\n\t" + "push {r1, %[length], lr}\n\t" + "ldr lr, [sp, #16]\n\t" + "rev r8, r4\n\t" + "add r8, r8, #1\n\t" + "rev r8, r8\n\t" + "mov r9, r5\n\t" + "mov r10, r6\n\t" + "mov r11, r7\n\t" + "stm lr, {r8, r9, r10, r11}\n\t" + "ldm %[KS]!, {r8, r9, r10, r11}\n\t" + /* Round: 0 - XOR in key schedule */ + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "mov r1, #6\n\t" +#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE + "bl AES_encrypt_block\n\t" #else - "ubfx r4, r11, #0, #8\n\t" -#endif + "\n" + "L_AES_GCMSIV_ctr_base_block_nr_256_%=:\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r10, #16\n\t" - "lsr r7, r7, #24\n\t" + "lsl r8, r5, #8\n\t" + "lsr r8, r8, #24\n\t" #else - "uxtb r7, r10, ror #8\n\t" + "uxtb r8, r5, ror #16\n\t" #endif #else - "ubfx r7, r10, #8, #8\n\t" + "ubfx r8, r5, #16, #8\n\t" #endif + "lsr r11, r4, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r9, #8\n\t" + "lsl lr, r6, #16\n\t" "lsr lr, lr, #24\n\t" #else - "uxtb lr, r9, ror #16\n\t" + "uxtb lr, r6, ror #8\n\t" #endif #else - "ubfx lr, r9, #16, #8\n\t" + "ubfx lr, r6, #8, #8\n\t" #endif - "lsr r2, r8, #24\n\t" - "ldrb r4, [r0, r4, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r8, #24\n\t" - "lsr r5, r5, #24\n\t" + "lsl r2, r7, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "uxtb r5, r8\n\t" + "uxtb r2, r7\n\t" #endif #else - "ubfx r5, r8, #0, #8\n\t" + "ubfx r2, r7, #0, #8\n\t" #endif - "eor r4, r4, r7, lsl #8\n\t" + "ldr r8, [r0, r8, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r11, #16\n\t" - "lsr r7, r7, #24\n\t" + "lsl r9, r6, #8\n\t" + "lsr r9, r9, #24\n\t" #else - "uxtb r7, r11, ror #8\n\t" + "uxtb r9, r6, ror #16\n\t" #endif #else - "ubfx r7, r11, #8, #8\n\t" + "ubfx r9, r6, #16, #8\n\t" #endif - "eor r4, r4, lr, lsl #16\n\t" + "eor r8, r8, r11, ror #24\n\t" + "lsr r11, r5, #24\n\t" + "eor r8, r8, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r10, #8\n\t" + "lsl lr, r7, #16\n\t" "lsr lr, lr, #24\n\t" #else - "uxtb lr, r10, ror #16\n\t" -#endif -#else - "ubfx lr, r10, #16, #8\n\t" -#endif - "eor r4, r4, r2, lsl #24\n\t" - "lsr r2, r9, #24\n\t" - "ldrb r5, [r0, r5, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r9, #24\n\t" - "lsr r6, r6, #24\n\t" -#else - "uxtb r6, r9\n\t" -#endif -#else - "ubfx r6, r9, #0, #8\n\t" -#endif - "eor r5, r5, r7, lsl #8\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r8, #16\n\t" - "lsr r7, r7, #24\n\t" -#else - "uxtb r7, r8, ror #8\n\t" + "uxtb lr, r7, ror #8\n\t" #endif #else - "ubfx r7, r8, #8, #8\n\t" + "ubfx lr, r7, #8, #8\n\t" #endif - "eor r5, r5, lr, lsl #16\n\t" + "eor r8, r8, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r11, #8\n\t" - "lsr lr, lr, #24\n\t" + "lsl r2, r4, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "uxtb lr, r11, ror #16\n\t" + "uxtb r2, r4\n\t" #endif #else - "ubfx lr, r11, #16, #8\n\t" + "ubfx r2, r4, #0, #8\n\t" #endif - "eor r5, r5, r2, lsl #24\n\t" - "lsr r2, r10, #24\n\t" - "ldrb r6, [r0, r6, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" - "lsr r11, r11, #24\n\t" - "eor r6, r6, r7, lsl #8\n\t" + "ldr r9, [r0, r9, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r10, #24\n\t" - "lsr r7, r7, #24\n\t" + "lsl r10, r7, #8\n\t" + "lsr r10, r10, #24\n\t" #else - "uxtb r7, r10\n\t" + "uxtb r10, r7, ror #16\n\t" #endif #else - "ubfx r7, r10, #0, #8\n\t" + "ubfx r10, r7, #16, #8\n\t" #endif - "eor r6, r6, lr, lsl #16\n\t" + "eor r9, r9, r11, ror #24\n\t" + "lsr r11, r6, #24\n\t" + "eor r9, r9, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r9, #16\n\t" + "lsl lr, r4, #16\n\t" "lsr lr, lr, #24\n\t" #else - "uxtb lr, r9, ror #8\n\t" + "uxtb lr, r4, ror #8\n\t" #endif #else - "ubfx lr, r9, #8, #8\n\t" + "ubfx lr, r4, #8, #8\n\t" #endif - "eor r6, r6, r2, lsl #24\n\t" + "eor r9, r9, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r8, #8\n\t" + "lsl r2, r5, #24\n\t" "lsr r2, r2, #24\n\t" #else - "uxtb r2, r8, ror #16\n\t" -#endif -#else - "ubfx r2, r8, #16, #8\n\t" + "uxtb r2, r5\n\t" #endif - "ldrb r11, [r0, r11, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r11, lsl #16\n\t" - "ldm %[ks], {r8, r9, r10, r11}\n\t" - "eor r7, r7, lr, lsl #8\n\t" - "eor r7, r7, r2, lsl #16\n\t" - /* XOR in Key Schedule */ - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" -#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - "pop {r1, %[len], lr}\n\t" - "ldr %[ks], [sp]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "eor r8, r4, r4, ror #16\n\t" - "eor r9, r5, r5, ror #16\n\t" - "eor r10, r6, r6, ror #16\n\t" - "eor r11, r7, r7, ror #16\n\t" - "bic r8, r8, #0xff0000\n\t" - "bic r9, r9, #0xff0000\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "ror r5, r5, #8\n\t" - "ror r6, r6, #8\n\t" - "ror r7, r7, #8\n\t" - "eor r4, r4, r8, lsr #8\n\t" - "eor r5, r5, r9, lsr #8\n\t" - "eor r6, r6, r10, lsr #8\n\t" - "eor r7, r7, r11, lsr #8\n\t" -#else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "ldr r8, [lr]\n\t" - "ldr r9, [lr, #4]\n\t" - "ldr r10, [lr, #8]\n\t" - "ldr r11, [lr, #12]\n\t" - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" - "ldr r8, [sp, #4]\n\t" - "str r4, [%[out]]\n\t" - "str r5, [%[out], #4]\n\t" - "str r6, [%[out], #8]\n\t" - "str r7, [%[out], #12]\n\t" - "ldm r8, {r4, r5, r6, r7}\n\t" - "subs %[len], %[len], #16\n\t" - "add lr, lr, #16\n\t" - "add %[out], %[out], #16\n\t" - "bne L_AES_GCM_encrypt_loop_block_128_%=\n\t" - "\n" - "L_AES_GCM_encrypt_end_%=:\n\t" - "pop {%[ks], r8}\n\t" +#else + "ubfx r2, r5, #0, #8\n\t" +#endif + "ldr r10, [r0, r10, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "eor r10, r4, r4, ror #16\n\t" - "eor r11, r5, r5, ror #16\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "ror r5, r5, #8\n\t" - "eor r4, r4, r10, lsr #8\n\t" - "eor r5, r5, r11, lsr #8\n\t" - "eor r10, r6, r6, ror #16\n\t" - "eor r11, r7, r7, ror #16\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r6, r6, #8\n\t" - "ror r7, r7, #8\n\t" - "eor r6, r6, r10, lsr #8\n\t" - "eor r7, r7, r11, lsr #8\n\t" + "lsl r6, r6, #24\n\t" + "lsr r6, r6, #24\n\t" #else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "stm r8, {r4, r5, r6, r7}\n\t" - "pop {%[nr], %[ctr]}\n\t" - "pop {%[L_AES_ARM32_te_gcm]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [in] "+r" (in), [out] "+r" (out), [len] "+r" (len), [ks] "+r" (ks), - [nr] "+r" (nr), [ctr] "+r" (ctr), - [L_AES_ARM32_te_gcm] "+r" (L_AES_ARM32_te_gcm_c) - : + "uxtb r6, r6\n\t" +#endif #else - : - : [in] "r" (in), [out] "r" (out), [len] "r" (len), [ks] "r" (ks), - [nr] "r" (nr), [ctr] "r" (ctr), - [L_AES_ARM32_te_gcm] "r" (L_AES_ARM32_te_gcm_c) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r5", "r6", "r7", "r8", "r9", "r10", "r11" - ); -} - -#endif /* HAVE_AESGCM */ -#ifdef WOLFSSL_AESGCM_SIV -XALIGNED(8) static const word32 L_AES_GCMSIV_polyval_base_r[] = { - 0x00000000, 0x1c200000, 0x38400000, 0x24600000, - 0x70800000, 0x6ca00000, 0x48c00000, 0x54e00000, - 0xe1000000, 0xfd200000, 0xd9400000, 0xc5600000, - 0x91800000, 0x8da00000, 0xa9c00000, 0xb5e00000, -}; - -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_GCMSIV_polyval_base(unsigned char* s_p, - const unsigned char* m_p, const unsigned char* data_p, - unsigned int blocks_p) + "ubfx r6, r6, #0, #8\n\t" +#endif + "eor r10, r10, r11, ror #24\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r11, r4, #8\n\t" + "lsr r11, r11, #24\n\t" #else -WC_OMIT_FRAME_POINTER void AES_GCMSIV_polyval_base(unsigned char* s, - const unsigned char* m, const unsigned char* data, unsigned int blocks) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register unsigned char* s __asm__ ("r0") = (unsigned char*)s_p; - register const unsigned char* m __asm__ ("r1") = (const unsigned char*)m_p; - register const unsigned char* data __asm__ ("r2") = - (const unsigned char*)data_p; - register unsigned int blocks __asm__ ("r3") = (unsigned int)blocks_p; - register word32* L_AES_ARM32_te_gcm_c __asm__ ("r12") = - (word32*)L_AES_ARM32_te_gcm; - register word32* L_AES_GCMSIV_polyval_base_r_c __asm__ ("lr") = - (word32*)&L_AES_GCMSIV_polyval_base_r; + "uxtb r11, r4, ror #16\n\t" +#endif #else - register word32* L_AES_ARM32_te_gcm_c = (word32*)L_AES_ARM32_te_gcm; - register word32* L_AES_GCMSIV_polyval_base_r_c = - (word32*)&L_AES_GCMSIV_polyval_base_r; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[L_AES_ARM32_te_gcm], %[L_AES_GCMSIV_polyval_base_r]}\n\t" - "mov r6, %[L_AES_GCMSIV_polyval_base_r]\n\t" - "cmp %[blocks], #0\n\t" - "beq L_AES_GCMSIV_polyval_base_done_%=\n\t" - "\n" - "L_AES_GCMSIV_polyval_base_loop_%=:\n\t" - "ldr r10, [%[data], #12]\n\t" - "rev r10, r10\n\t" - "ldr r8, [%[s]]\n\t" - "eor r8, r8, r10\n\t" - "str r8, [%[s]]\n\t" - "ldr r10, [%[data], #8]\n\t" - "rev r10, r10\n\t" - "ldr r8, [%[s], #4]\n\t" - "eor r8, r8, r10\n\t" - "str r8, [%[s], #4]\n\t" - "ldr r10, [%[data], #4]\n\t" - "rev r10, r10\n\t" - "ldr r8, [%[s], #8]\n\t" - "eor r8, r8, r10\n\t" - "str r8, [%[s], #8]\n\t" - "ldr r10, [%[data]]\n\t" - "rev r10, r10\n\t" - "ldr r8, [%[s], #12]\n\t" - "eor r8, r8, r10\n\t" - "str r8, [%[s], #12]\n\t" - "mov r12, #0\n\t" - "mov lr, #0\n\t" - "mov r4, #0\n\t" - "mov r5, #0\n\t" - "ldr r7, [%[s], #12]\n\t" + "ubfx r11, r4, #16, #8\n\t" +#endif + "eor r10, r10, lr, ror #8\n\t" + "lsr lr, r7, #24\n\t" + "eor r10, r10, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #4\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r5, #16\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r8, r7, #24, #4\n\t" + "uxtb r2, r5, ror #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r2, r5, #8, #8\n\t" +#endif + "ldr r6, [r0, r6, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r6, ror #24\n\t" + "ldm %[KS]!, {r4, r5, r6, r7}\n\t" + "eor r11, r11, lr, ror #24\n\t" + "eor r11, r11, r2, ror #8\n\t" + /* XOR in Key Schedule */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsr r8, r7, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r9, #8\n\t" + "lsr r4, r4, #24\n\t" #else - "ubfx r8, r7, #28, #4\n\t" + "uxtb r4, r9, ror #16\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r4, r9, #16, #8\n\t" +#endif + "lsr r7, r8, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #12\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r10, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r8, r7, #16, #4\n\t" + "uxtb lr, r10, ror #8\n\t" +#endif +#else + "ubfx lr, r10, #8, #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #8\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r11, #24\n\t" + "lsr r2, r2, #24\n\t" +#else + "uxtb r2, r11\n\t" +#endif +#else + "ubfx r2, r11, #0, #8\n\t" +#endif + "ldr r4, [r0, r4, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r5, r10, #8\n\t" + "lsr r5, r5, #24\n\t" +#else + "uxtb r5, r10, ror #16\n\t" +#endif +#else + "ubfx r5, r10, #16, #8\n\t" +#endif + "eor r4, r4, r7, ror #24\n\t" + "lsr r7, r9, #24\n\t" + "eor r4, r4, lr, ror #8\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r11, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r8, r7, #20, #4\n\t" + "uxtb lr, r11, ror #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx lr, r11, #8, #8\n\t" +#endif + "eor r4, r4, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #20\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r8, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r8, r7, #8, #4\n\t" + "uxtb r2, r8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r2, r8, #0, #8\n\t" +#endif + "ldr r5, [r0, r5, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #16\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r11, #8\n\t" + "lsr r6, r6, #24\n\t" #else - "ubfx r8, r7, #12, #4\n\t" + "uxtb r6, r11, ror #16\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r6, r11, #16, #8\n\t" +#endif + "eor r5, r5, r7, ror #24\n\t" + "lsr r7, r10, #24\n\t" + "eor r5, r5, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #28\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r8, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r8, r7, #0, #4\n\t" + "uxtb lr, r8, ror #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx lr, r8, #8, #8\n\t" +#endif + "eor r5, r5, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #24\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r9, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r8, r7, #4, #4\n\t" + "uxtb r2, r9\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" - "ldr r7, [%[s], #8]\n\t" +#else + "ubfx r2, r9, #0, #8\n\t" +#endif + "ldr r6, [r0, r6, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #4\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r10, r10, #24\n\t" + "lsr r10, r10, #24\n\t" #else - "ubfx r8, r7, #24, #4\n\t" + "uxtb r10, r10\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r10, r10, #0, #8\n\t" +#endif + "eor r6, r6, r7, ror #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsr r8, r7, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r8, #8\n\t" + "lsr r7, r7, #24\n\t" #else - "ubfx r8, r7, #28, #4\n\t" + "uxtb r7, r8, ror #16\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r7, r8, #16, #8\n\t" +#endif + "eor r6, r6, lr, ror #8\n\t" + "lsr lr, r11, #24\n\t" + "eor r6, r6, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #12\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r9, #16\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r8, r7, #16, #4\n\t" + "uxtb r2, r9, ror #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r2, r9, #8, #8\n\t" +#endif + "ldr r10, [r0, r10, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r7, [r0, r7, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r10, ror #24\n\t" + "ldm %[KS]!, {r8, r9, r10, r11}\n\t" + "eor r7, r7, lr, ror #24\n\t" + "eor r7, r7, r2, ror #8\n\t" + /* XOR in Key Schedule */ + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "subs r1, r1, #1\n\t" + "bne L_AES_GCMSIV_ctr_base_block_nr_256_%=\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #8\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r8, r5, #8\n\t" + "lsr r8, r8, #24\n\t" #else - "ubfx r8, r7, #20, #4\n\t" + "uxtb r8, r5, ror #16\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r8, r5, #16, #8\n\t" +#endif + "lsr r11, r4, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #20\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r6, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r8, r7, #8, #4\n\t" + "uxtb lr, r6, ror #8\n\t" +#endif +#else + "ubfx lr, r6, #8, #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #16\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r7, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r8, r7, #12, #4\n\t" + "uxtb r2, r7\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r2, r7, #0, #8\n\t" +#endif + "ldr r8, [r0, r8, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #28\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r9, r6, #8\n\t" + "lsr r9, r9, #24\n\t" #else - "ubfx r8, r7, #0, #4\n\t" + "uxtb r9, r6, ror #16\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r9, r6, #16, #8\n\t" +#endif + "eor r8, r8, r11, ror #24\n\t" + "lsr r11, r5, #24\n\t" + "eor r8, r8, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #24\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r7, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r8, r7, #4, #4\n\t" + "uxtb lr, r7, ror #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" - "ldr r7, [%[s], #4]\n\t" +#else + "ubfx lr, r7, #8, #8\n\t" +#endif + "eor r8, r8, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #4\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r4, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r8, r7, #24, #4\n\t" + "uxtb r2, r4\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r2, r4, #0, #8\n\t" +#endif + "ldr r9, [r0, r9, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsr r8, r7, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r10, r7, #8\n\t" + "lsr r10, r10, #24\n\t" #else - "ubfx r8, r7, #28, #4\n\t" + "uxtb r10, r7, ror #16\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r10, r7, #16, #8\n\t" +#endif + "eor r9, r9, r11, ror #24\n\t" + "lsr r11, r6, #24\n\t" + "eor r9, r9, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #12\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r4, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r8, r7, #16, #4\n\t" + "uxtb lr, r4, ror #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx lr, r4, #8, #8\n\t" +#endif + "eor r9, r9, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #8\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r5, #24\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r8, r7, #20, #4\n\t" + "uxtb r2, r5\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #20\n\t" - "lsr r8, r8, #28\n\t" #else - "ubfx r8, r7, #8, #4\n\t" + "ubfx r2, r5, #0, #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" + "ldr r10, [r0, r10, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #16\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r6, #24\n\t" + "lsr r6, r6, #24\n\t" #else - "ubfx r8, r7, #12, #4\n\t" + "uxtb r6, r6\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r6, r6, #0, #8\n\t" +#endif + "eor r10, r10, r11, ror #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #28\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r11, r4, #8\n\t" + "lsr r11, r11, #24\n\t" #else - "ubfx r8, r7, #0, #4\n\t" + "uxtb r11, r4, ror #16\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r11, r4, #16, #8\n\t" +#endif + "eor r10, r10, lr, ror #8\n\t" + "lsr lr, r7, #24\n\t" + "eor r10, r10, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #24\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r5, #16\n\t" + "lsr r2, r2, #24\n\t" #else - "ubfx r8, r7, #4, #4\n\t" + "uxtb r2, r5, ror #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" - "ldr r7, [%[s]]\n\t" +#else + "ubfx r2, r5, #8, #8\n\t" +#endif + "ldr r6, [r0, r6, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r6, ror #24\n\t" + "ldm %[KS]!, {r4, r5, r6, r7}\n\t" + "eor r11, r11, lr, ror #24\n\t" + "eor r11, r11, r2, ror #8\n\t" + /* XOR in Key Schedule */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #4\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r11, #24\n\t" + "lsr r4, r4, #24\n\t" #else - "ubfx r8, r7, #24, #4\n\t" + "uxtb r4, r11\n\t" +#endif +#else + "ubfx r4, r11, #0, #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsr r8, r7, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r10, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "ubfx r8, r7, #28, #4\n\t" + "uxtb r7, r10, ror #8\n\t" +#endif +#else + "ubfx r7, r10, #8, #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #12\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r9, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r8, r7, #16, #4\n\t" + "uxtb lr, r9, ror #16\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx lr, r9, #16, #8\n\t" +#endif + "lsr r2, r8, #24\n\t" + "ldrb r4, [r0, r4, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r5, r8, #24\n\t" + "lsr r5, r5, #24\n\t" +#else + "uxtb r5, r8\n\t" +#endif +#else + "ubfx r5, r8, #0, #8\n\t" +#endif + "eor r4, r4, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #8\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r11, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "ubfx r8, r7, #20, #4\n\t" + "uxtb r7, r11, ror #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r7, r11, #8, #8\n\t" +#endif + "eor r4, r4, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #20\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r10, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r8, r7, #8, #4\n\t" + "uxtb lr, r10, ror #16\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx lr, r10, #16, #8\n\t" +#endif + "eor r4, r4, r2, lsl #24\n\t" + "lsr r2, r9, #24\n\t" + "ldrb r5, [r0, r5, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #16\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r9, #24\n\t" + "lsr r6, r6, #24\n\t" #else - "ubfx r8, r7, #12, #4\n\t" + "uxtb r6, r9\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r6, r9, #0, #8\n\t" +#endif + "eor r5, r5, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #28\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r8, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "ubfx r8, r7, #0, #4\n\t" + "uxtb r7, r8, ror #8\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "and r8, r4, #15\n\t" - "lsr r4, r4, #4\n\t" - "orr r4, r4, r5, lsl #28\n\t" - "lsr r5, r5, #4\n\t" - "orr r5, r5, r12, lsl #28\n\t" - "lsr r12, r12, #4\n\t" - "orr r12, r12, lr, lsl #28\n\t" - "lsr lr, lr, #4\n\t" - "ldr r10, [r6, r8, lsl #2]\n\t" - "eor lr, lr, r10\n\t" +#else + "ubfx r7, r8, #8, #8\n\t" +#endif + "eor r5, r5, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) - "lsl r8, r7, #24\n\t" - "lsr r8, r8, #28\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r11, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "ubfx r8, r7, #4, #4\n\t" + "uxtb lr, r11, ror #16\n\t" #endif - "add r9, %[m], r8, lsl #4\n\t" - "ldr r10, [r9]\n\t" - "eor r12, r12, r10\n\t" - "ldr r10, [r9, #4]\n\t" - "eor lr, lr, r10\n\t" - "ldr r10, [r9, #8]\n\t" - "eor r4, r4, r10\n\t" - "ldr r10, [r9, #12]\n\t" - "eor r5, r5, r10\n\t" - "rev lr, lr\n\t" - "rev r12, r12\n\t" - "rev r5, r5\n\t" - "rev r4, r4\n\t" - "str lr, [%[s]]\n\t" - "str r12, [%[s], #4]\n\t" - "str r5, [%[s], #8]\n\t" - "str r4, [%[s], #12]\n\t" - "subs %[blocks], %[blocks], #1\n\t" - "add %[data], %[data], #16\n\t" - "bne L_AES_GCMSIV_polyval_base_loop_%=\n\t" - "\n" - "L_AES_GCMSIV_polyval_base_done_%=:\n\t" - "pop {%[L_AES_ARM32_te_gcm], %[L_AES_GCMSIV_polyval_base_r]}\n\t" -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - : [s] "+r" (s), [m] "+r" (m), [data] "+r" (data), - [blocks] "+r" (blocks), - [L_AES_ARM32_te_gcm] "+r" (L_AES_ARM32_te_gcm_c), - [L_AES_GCMSIV_polyval_base_r] "+r" (L_AES_GCMSIV_polyval_base_r_c) - : #else - : - : [s] "r" (s), [m] "r" (m), [data] "r" (data), [blocks] "r" (blocks), - [L_AES_ARM32_te_gcm] "r" (L_AES_ARM32_te_gcm_c), - [L_AES_GCMSIV_polyval_base_r] "r" (L_AES_GCMSIV_polyval_base_r_c) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r4", "r5", "r6", "r7", "r8", "r9", "r10" - ); -} - -XALIGNED(8) static const word32 L_AES_GCMSIV_ctr_base_te_data[] = { - 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, - 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, - 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, - 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, - 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, - 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, - 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, - 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, - 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, - 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, - 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, - 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, - 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, - 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, - 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, - 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, - 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, - 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, - 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, - 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, - 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, - 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, - 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, - 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, - 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, - 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, - 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, - 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, - 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, - 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, - 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, - 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, - 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, - 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, - 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, - 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, - 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, - 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, - 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, - 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, - 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, - 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, - 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, - 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, - 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, - 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, - 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, - 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, - 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, - 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, - 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, - 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, - 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, - 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, - 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, - 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, - 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, - 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, - 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, - 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, - 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, - 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, - 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, - 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, -}; - -static const word32* L_AES_GCMSIV_ctr_base_te = L_AES_GCMSIV_ctr_base_te_data; -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in_p, - unsigned char* out_p, unsigned long length_p, const unsigned char* KS_p, - int nr_p, unsigned char* ctr_p) + "ubfx lr, r11, #16, #8\n\t" +#endif + "eor r5, r5, r2, lsl #24\n\t" + "lsr r2, r10, #24\n\t" + "ldrb r6, [r0, r6, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" + "lsr r11, r11, #24\n\t" + "eor r6, r6, r7, lsl #8\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r10, #24\n\t" + "lsr r7, r7, #24\n\t" #else -WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in, - unsigned char* out, unsigned long length, const unsigned char* KS, int nr, - unsigned char* ctr) -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ -{ -#ifndef WOLFSSL_NO_VAR_ASSIGN_REG - register const unsigned char* in __asm__ ("r0") = - (const unsigned char*)in_p; - register unsigned char* out __asm__ ("r1") = (unsigned char*)out_p; - register unsigned long length __asm__ ("r2") = (unsigned long)length_p; - register const unsigned char* KS __asm__ ("r3") = - (const unsigned char*)KS_p; - register int nr __asm__ ("r12") = (int)nr_p; - register unsigned char* ctr __asm__ ("lr") = (unsigned char*)ctr_p; - register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r4") = - (word32*)L_AES_GCMSIV_ctr_base_te; + "uxtb r7, r10\n\t" +#endif #else - register word32* L_AES_GCMSIV_ctr_base_te_c = - (word32*)L_AES_GCMSIV_ctr_base_te; -#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - - __asm__ __volatile__ ( - "push {%[L_AES_GCMSIV_ctr_base_te]}\n\t" - "push {%[nr], %[ctr]}\n\t" - "ldr r12, [sp]\n\t" - "ldr r8, [sp, #4]\n\t" - "mov lr, %[in]\n\t" - "ldr r0, [sp, #8]\n\t" - "ldm r8, {r4, r5, r6, r7}\n\t" + "ubfx r7, r10, #0, #8\n\t" +#endif + "eor r6, r6, lr, lsl #16\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "eor r10, r4, r4, ror #16\n\t" - "eor r11, r5, r5, ror #16\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "ror r5, r5, #8\n\t" - "eor r4, r4, r10, lsr #8\n\t" - "eor r5, r5, r11, lsr #8\n\t" + "lsl lr, r9, #16\n\t" + "lsr lr, lr, #24\n\t" +#else + "uxtb lr, r9, ror #8\n\t" +#endif +#else + "ubfx lr, r9, #8, #8\n\t" +#endif + "eor r6, r6, r2, lsl #24\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r8, #8\n\t" + "lsr r2, r2, #24\n\t" +#else + "uxtb r2, r8, ror #16\n\t" +#endif +#else + "ubfx r2, r8, #16, #8\n\t" +#endif + "ldrb r11, [r0, r11, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r11, lsl #16\n\t" + "ldm %[KS], {r8, r9, r10, r11}\n\t" + "eor r7, r7, lr, lsl #8\n\t" + "eor r7, r7, r2, lsl #16\n\t" + /* XOR in Key Schedule */ + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" +#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ + "pop {r1, %[length], lr}\n\t" + "ldr %[KS], [sp]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "eor r8, r4, r4, ror #16\n\t" + "eor r9, r5, r5, ror #16\n\t" "eor r10, r6, r6, ror #16\n\t" "eor r11, r7, r7, ror #16\n\t" + "bic r8, r8, #0xff0000\n\t" + "bic r9, r9, #0xff0000\n\t" "bic r10, r10, #0xff0000\n\t" "bic r11, r11, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "ror r5, r5, #8\n\t" "ror r6, r6, #8\n\t" "ror r7, r7, #8\n\t" + "eor r4, r4, r8, lsr #8\n\t" + "eor r5, r5, r9, lsr #8\n\t" "eor r6, r6, r10, lsr #8\n\t" "eor r7, r7, r11, lsr #8\n\t" #else @@ -28393,14 +20295,29 @@ WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in, "rev r6, r6\n\t" "rev r7, r7\n\t" #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "stm r8, {r4, r5, r6, r7}\n\t" - "push {%[KS], r8}\n\t" - "cmp r12, #10\n\t" - "beq L_AES_GCMSIV_ctr_base_start_block_128_%=\n\t" - "cmp r12, #12\n\t" - "beq L_AES_GCMSIV_ctr_base_start_block_192_%=\n\t" + "ldr r8, [lr]\n\t" + "ldr r9, [lr, #4]\n\t" + "ldr r10, [lr, #8]\n\t" + "ldr r11, [lr, #12]\n\t" + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "ldr r8, [sp, #4]\n\t" + "str r4, [%[out]]\n\t" + "str r5, [%[out], #4]\n\t" + "str r6, [%[out], #8]\n\t" + "str r7, [%[out], #12]\n\t" + "ldm r8, {r4, r5, r6, r7}\n\t" + "subs %[length], %[length], #16\n\t" + "add lr, lr, #16\n\t" + "add %[out], %[out], #16\n\t" + "bne L_AES_GCMSIV_ctr_base_loop_block_256_%=\n\t" + "b L_AES_GCMSIV_ctr_base_end_%=\n\t" "\n" - "L_AES_GCMSIV_ctr_base_loop_block_256_%=:\n\t" + "L_AES_GCMSIV_ctr_base_start_block_192_%=:\n\t" + "\n" + "L_AES_GCMSIV_ctr_base_loop_block_192_%=:\n\t" "push {r1, %[length], lr}\n\t" "ldr lr, [sp, #16]\n\t" "rev r8, r4\n\t" @@ -28416,12 +20333,12 @@ WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in, "eor r5, r5, r9\n\t" "eor r6, r6, r10\n\t" "eor r7, r7, r11\n\t" - "mov r1, #6\n\t" + "mov r1, #5\n\t" #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE "bl AES_encrypt_block\n\t" #else "\n" - "L_AES_GCMSIV_ctr_base_block_nr_256_%=:\n\t" + "L_AES_GCMSIV_ctr_base_block_nr_192_%=:\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) "lsl r8, r5, #8\n\t" @@ -28739,7 +20656,7 @@ WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in, "eor r6, r6, r10\n\t" "eor r7, r7, r11\n\t" "subs r1, r1, #1\n\t" - "bne L_AES_GCMSIV_ctr_base_block_nr_256_%=\n\t" + "bne L_AES_GCMSIV_ctr_base_block_nr_192_%=\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) "lsl r8, r5, #8\n\t" @@ -29099,12 +21016,12 @@ WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in, "subs %[length], %[length], #16\n\t" "add lr, lr, #16\n\t" "add %[out], %[out], #16\n\t" - "bne L_AES_GCMSIV_ctr_base_loop_block_256_%=\n\t" + "bne L_AES_GCMSIV_ctr_base_loop_block_192_%=\n\t" "b L_AES_GCMSIV_ctr_base_end_%=\n\t" "\n" - "L_AES_GCMSIV_ctr_base_start_block_192_%=:\n\t" + "L_AES_GCMSIV_ctr_base_start_block_128_%=:\n\t" "\n" - "L_AES_GCMSIV_ctr_base_loop_block_192_%=:\n\t" + "L_AES_GCMSIV_ctr_base_loop_block_128_%=:\n\t" "push {r1, %[length], lr}\n\t" "ldr lr, [sp, #16]\n\t" "rev r8, r4\n\t" @@ -29120,12 +21037,12 @@ WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in, "eor r5, r5, r9\n\t" "eor r6, r6, r10\n\t" "eor r7, r7, r11\n\t" - "mov r1, #5\n\t" + "mov r1, #4\n\t" #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE "bl AES_encrypt_block\n\t" #else "\n" - "L_AES_GCMSIV_ctr_base_block_nr_192_%=:\n\t" + "L_AES_GCMSIV_ctr_base_block_nr_128_%=:\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) "lsl r8, r5, #8\n\t" @@ -29443,7 +21360,7 @@ WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in, "eor r6, r6, r10\n\t" "eor r7, r7, r11\n\t" "subs r1, r1, #1\n\t" - "bne L_AES_GCMSIV_ctr_base_block_nr_192_%=\n\t" + "bne L_AES_GCMSIV_ctr_base_block_nr_128_%=\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) "lsl r8, r5, #8\n\t" @@ -29538,1003 +21455,9206 @@ WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in, "uxtb lr, r4, ror #8\n\t" #endif #else - "ubfx lr, r4, #8, #8\n\t" + "ubfx lr, r4, #8, #8\n\t" +#endif + "eor r9, r9, r2, ror #16\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r5, #24\n\t" + "lsr r2, r2, #24\n\t" +#else + "uxtb r2, r5\n\t" +#endif +#else + "ubfx r2, r5, #0, #8\n\t" +#endif + "ldr r10, [r0, r10, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r6, r6, #24\n\t" + "lsr r6, r6, #24\n\t" +#else + "uxtb r6, r6\n\t" +#endif +#else + "ubfx r6, r6, #0, #8\n\t" +#endif + "eor r10, r10, r11, ror #24\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r11, r4, #8\n\t" + "lsr r11, r11, #24\n\t" +#else + "uxtb r11, r4, ror #16\n\t" +#endif +#else + "ubfx r11, r4, #16, #8\n\t" +#endif + "eor r10, r10, lr, ror #8\n\t" + "lsr lr, r7, #24\n\t" + "eor r10, r10, r2, ror #16\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r2, r5, #16\n\t" + "lsr r2, r2, #24\n\t" +#else + "uxtb r2, r5, ror #8\n\t" +#endif +#else + "ubfx r2, r5, #8, #8\n\t" +#endif + "ldr r6, [r0, r6, lsl #2]\n\t" + "ldr lr, [r0, lr, lsl #2]\n\t" + "ldr r11, [r0, r11, lsl #2]\n\t" + "ldr r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r6, ror #24\n\t" + "ldm %[KS]!, {r4, r5, r6, r7}\n\t" + "eor r11, r11, lr, ror #24\n\t" + "eor r11, r11, r2, ror #8\n\t" + /* XOR in Key Schedule */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r4, r11, #24\n\t" + "lsr r4, r4, #24\n\t" +#else + "uxtb r4, r11\n\t" +#endif +#else + "ubfx r4, r11, #0, #8\n\t" +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r10, #16\n\t" + "lsr r7, r7, #24\n\t" +#else + "uxtb r7, r10, ror #8\n\t" +#endif +#else + "ubfx r7, r10, #8, #8\n\t" +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl lr, r9, #8\n\t" + "lsr lr, lr, #24\n\t" +#else + "uxtb lr, r9, ror #16\n\t" +#endif +#else + "ubfx lr, r9, #16, #8\n\t" +#endif + "lsr r2, r8, #24\n\t" + "ldrb r4, [r0, r4, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r5, r8, #24\n\t" + "lsr r5, r5, #24\n\t" +#else + "uxtb r5, r8\n\t" +#endif +#else + "ubfx r5, r8, #0, #8\n\t" +#endif + "eor r4, r4, r7, lsl #8\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "lsl r7, r11, #16\n\t" + "lsr r7, r7, #24\n\t" +#else + "uxtb r7, r11, ror #8\n\t" +#endif +#else + "ubfx r7, r11, #8, #8\n\t" #endif - "eor r9, r9, r2, ror #16\n\t" + "eor r4, r4, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #24\n\t" - "lsr r2, r2, #24\n\t" + "lsl lr, r10, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r2, r5\n\t" + "uxtb lr, r10, ror #16\n\t" #endif #else - "ubfx r2, r5, #0, #8\n\t" + "ubfx lr, r10, #16, #8\n\t" #endif - "ldr r10, [r0, r10, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "eor r4, r4, r2, lsl #24\n\t" + "lsr r2, r9, #24\n\t" + "ldrb r5, [r0, r5, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r6, #24\n\t" + "lsl r6, r9, #24\n\t" "lsr r6, r6, #24\n\t" #else - "uxtb r6, r6\n\t" + "uxtb r6, r9\n\t" #endif #else - "ubfx r6, r6, #0, #8\n\t" + "ubfx r6, r9, #0, #8\n\t" #endif - "eor r10, r10, r11, ror #24\n\t" + "eor r5, r5, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r11, r4, #8\n\t" - "lsr r11, r11, #24\n\t" + "lsl r7, r8, #16\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb r11, r4, ror #16\n\t" + "uxtb r7, r8, ror #8\n\t" #endif #else - "ubfx r11, r4, #16, #8\n\t" + "ubfx r7, r8, #8, #8\n\t" #endif - "eor r10, r10, lr, ror #8\n\t" - "lsr lr, r7, #24\n\t" - "eor r10, r10, r2, ror #16\n\t" + "eor r5, r5, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #16\n\t" - "lsr r2, r2, #24\n\t" + "lsl lr, r11, #8\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r2, r5, ror #8\n\t" + "uxtb lr, r11, ror #16\n\t" #endif #else - "ubfx r2, r5, #8, #8\n\t" + "ubfx lr, r11, #16, #8\n\t" #endif - "ldr r6, [r0, r6, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r6, ror #24\n\t" - "ldm %[KS]!, {r4, r5, r6, r7}\n\t" - "eor r11, r11, lr, ror #24\n\t" - "eor r11, r11, r2, ror #8\n\t" - /* XOR in Key Schedule */ - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" + "eor r5, r5, r2, lsl #24\n\t" + "lsr r2, r10, #24\n\t" + "ldrb r6, [r0, r6, lsl #2]\n\t" + "ldrb r7, [r0, r7, lsl #2]\n\t" + "ldrb lr, [r0, lr, lsl #2]\n\t" + "ldrb r2, [r0, r2, lsl #2]\n\t" + "lsr r11, r11, #24\n\t" + "eor r6, r6, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r11, #24\n\t" - "lsr r4, r4, #24\n\t" + "lsl r7, r10, #24\n\t" + "lsr r7, r7, #24\n\t" #else - "uxtb r4, r11\n\t" + "uxtb r7, r10\n\t" #endif #else - "ubfx r4, r11, #0, #8\n\t" + "ubfx r7, r10, #0, #8\n\t" #endif + "eor r6, r6, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r10, #16\n\t" - "lsr r7, r7, #24\n\t" + "lsl lr, r9, #16\n\t" + "lsr lr, lr, #24\n\t" #else - "uxtb r7, r10, ror #8\n\t" + "uxtb lr, r9, ror #8\n\t" #endif #else - "ubfx r7, r10, #8, #8\n\t" + "ubfx lr, r9, #8, #8\n\t" #endif + "eor r6, r6, r2, lsl #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r9, #8\n\t" - "lsr lr, lr, #24\n\t" + "lsl r2, r8, #8\n\t" + "lsr r2, r2, #24\n\t" #else - "uxtb lr, r9, ror #16\n\t" + "uxtb r2, r8, ror #16\n\t" #endif #else - "ubfx lr, r9, #16, #8\n\t" + "ubfx r2, r8, #16, #8\n\t" #endif - "lsr r2, r8, #24\n\t" - "ldrb r4, [r0, r4, lsl #2]\n\t" + "ldrb r11, [r0, r11, lsl #2]\n\t" "ldrb r7, [r0, r7, lsl #2]\n\t" "ldrb lr, [r0, lr, lsl #2]\n\t" "ldrb r2, [r0, r2, lsl #2]\n\t" + "eor lr, lr, r11, lsl #16\n\t" + "ldm %[KS], {r8, r9, r10, r11}\n\t" + "eor r7, r7, lr, lsl #8\n\t" + "eor r7, r7, r2, lsl #16\n\t" + /* XOR in Key Schedule */ + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" +#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ + "pop {r1, %[length], lr}\n\t" + "ldr %[KS], [sp]\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "eor r8, r4, r4, ror #16\n\t" + "eor r9, r5, r5, ror #16\n\t" + "eor r10, r6, r6, ror #16\n\t" + "eor r11, r7, r7, ror #16\n\t" + "bic r8, r8, #0xff0000\n\t" + "bic r9, r9, #0xff0000\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "ror r5, r5, #8\n\t" + "ror r6, r6, #8\n\t" + "ror r7, r7, #8\n\t" + "eor r4, r4, r8, lsr #8\n\t" + "eor r5, r5, r9, lsr #8\n\t" + "eor r6, r6, r10, lsr #8\n\t" + "eor r7, r7, r11, lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "ldr r8, [lr]\n\t" + "ldr r9, [lr, #4]\n\t" + "ldr r10, [lr, #8]\n\t" + "ldr r11, [lr, #12]\n\t" + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "ldr r8, [sp, #4]\n\t" + "str r4, [%[out]]\n\t" + "str r5, [%[out], #4]\n\t" + "str r6, [%[out], #8]\n\t" + "str r7, [%[out], #12]\n\t" + "ldm r8, {r4, r5, r6, r7}\n\t" + "subs %[length], %[length], #16\n\t" + "add lr, lr, #16\n\t" + "add %[out], %[out], #16\n\t" + "bne L_AES_GCMSIV_ctr_base_loop_block_128_%=\n\t" + "\n" + "L_AES_GCMSIV_ctr_base_end_%=:\n\t" + "pop {%[KS], r8}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + "eor r10, r4, r4, ror #16\n\t" + "eor r11, r5, r5, ror #16\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "ror r5, r5, #8\n\t" + "eor r4, r4, r10, lsr #8\n\t" + "eor r5, r5, r11, lsr #8\n\t" + "eor r10, r6, r6, ror #16\n\t" + "eor r11, r7, r7, ror #16\n\t" + "bic r10, r10, #0xff0000\n\t" + "bic r11, r11, #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "ror r7, r7, #8\n\t" + "eor r6, r6, r10, lsr #8\n\t" + "eor r7, r7, r11, lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "stm r8, {r4, r5, r6, r7}\n\t" + "pop {%[nr], %[ctr]}\n\t" + "pop {%[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [length] "+r" (length), + [KS] "+r" (KS), [nr] "+r" (nr), [ctr] "+r" (ctr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : +#else + : + : [in] "r" (in), [out] "r" (out), [length] "r" (length), [KS] "r" (KS), + [nr] "r" (nr), [ctr] "r" (ctr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r5", "r6", "r7", "r8", "r9", "r10", "r11" + ); +} + +#endif /* WOLFSSL_AESGCM_SIV */ +#endif /* !WOLFSSL_ARMASM_NO_BASE_IMPL || WOLFSSL_ARMASM_NO_HW_CRYPTO */ +#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_set_key_AARCH32(const byte* userKey_p, + int keylen_p, byte* key_p, int dir_p) +#else +WC_OMIT_FRAME_POINTER void AES_set_key_AARCH32(const byte* userKey, int keylen, + byte* key, int dir) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* userKey __asm__ ("r0") = (const byte*)userKey_p; + register int keylen __asm__ ("r1") = (int)keylen_p; + register byte* key __asm__ ("r2") = (byte*)key_p; + register int dir __asm__ ("r3") = (int)dir_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r12") = + (word32*)L_AES_GCMSIV_ctr_base_te; +#else + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_GCMSIV_ctr_base_te]}\n\t" + "cmp %[keylen], #24\n\t" + "blt L_aes_set_key_arm32_crypto_start_128_%=\n\t" + "bgt L_aes_set_key_arm32_crypto_start_256_%=\n\t" + "ldr r4, [%[userKey]], #4\n\t" + "ldr r5, [%[userKey]], #4\n\t" + "ldr r6, [%[userKey]], #4\n\t" + "ldr r7, [%[userKey]], #4\n\t" + "ldr r8, [%[userKey]], #4\n\t" + "ldr r9, [%[userKey]], #4\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" +#else + "strd r4, r5, [%[key]], #8\n\t" +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" +#else + "strd r6, r7, [%[key]], #8\n\t" +#endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r8, r9}\n\t" +#else + "strd r8, r9, [%[key]], #8\n\t" +#endif + "vdup.32 q1, r9\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #1\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "eor r8, r8, r7\n\t" + "eor r9, r9, r8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r8, #24\n\t" - "lsr r5, r5, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb r5, r8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx r5, r8, #0, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "eor r4, r4, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r11, #16\n\t" - "lsr r7, r7, #24\n\t" + "stm r2!, {r8, r9}\n\t" #else - "uxtb r7, r11, ror #8\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif + "vdup.32 q1, r9\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #2\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "eor r8, r8, r7\n\t" + "eor r9, r9, r8\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx r7, r11, #8, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "eor r4, r4, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r10, #8\n\t" - "lsr lr, lr, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb lr, r10, ror #16\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r8, r9}\n\t" #else - "ubfx lr, r10, #16, #8\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif - "eor r4, r4, r2, lsl #24\n\t" - "lsr r2, r9, #24\n\t" - "ldrb r5, [r0, r5, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" + "vdup.32 q1, r9\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #4\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "eor r8, r8, r7\n\t" + "eor r9, r9, r8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r9, #24\n\t" - "lsr r6, r6, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb r6, r9\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx r6, r9, #0, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "eor r5, r5, r7, lsl #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r8, #16\n\t" - "lsr r7, r7, #24\n\t" + "stm r2!, {r8, r9}\n\t" #else - "uxtb r7, r8, ror #8\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif + "vdup.32 q1, r9\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #8\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "eor r8, r8, r7\n\t" + "eor r9, r9, r8\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx r7, r8, #8, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "eor r5, r5, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r11, #8\n\t" - "lsr lr, lr, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb lr, r11, ror #16\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r8, r9}\n\t" #else - "ubfx lr, r11, #16, #8\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif - "eor r5, r5, r2, lsl #24\n\t" - "lsr r2, r10, #24\n\t" - "ldrb r6, [r0, r6, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" - "lsr r11, r11, #24\n\t" - "eor r6, r6, r7, lsl #8\n\t" + "vdup.32 q1, r9\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #16\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "eor r8, r8, r7\n\t" + "eor r9, r9, r8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r10, #24\n\t" - "lsr r7, r7, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb r7, r10\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx r7, r10, #0, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "eor r6, r6, lr, lsl #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r9, #16\n\t" - "lsr lr, lr, #24\n\t" + "stm r2!, {r8, r9}\n\t" #else - "uxtb lr, r9, ror #8\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif + "vdup.32 q1, r9\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #32\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "eor r8, r8, r7\n\t" + "eor r9, r9, r8\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx lr, r9, #8, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "eor r6, r6, r2, lsl #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r8, #8\n\t" - "lsr r2, r2, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb r2, r8, ror #16\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r8, r9}\n\t" #else - "ubfx r2, r8, #16, #8\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif - "ldrb r11, [r0, r11, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r11, lsl #16\n\t" - "ldm %[KS], {r8, r9, r10, r11}\n\t" - "eor r7, r7, lr, lsl #8\n\t" - "eor r7, r7, r2, lsl #16\n\t" - /* XOR in Key Schedule */ - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" -#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - "pop {r1, %[length], lr}\n\t" - "ldr %[KS], [sp]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "eor r8, r4, r4, ror #16\n\t" - "eor r9, r5, r5, ror #16\n\t" - "eor r10, r6, r6, ror #16\n\t" - "eor r11, r7, r7, ror #16\n\t" - "bic r8, r8, #0xff0000\n\t" - "bic r9, r9, #0xff0000\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "ror r5, r5, #8\n\t" - "ror r6, r6, #8\n\t" - "ror r7, r7, #8\n\t" - "eor r4, r4, r8, lsr #8\n\t" - "eor r5, r5, r9, lsr #8\n\t" - "eor r6, r6, r10, lsr #8\n\t" - "eor r7, r7, r11, lsr #8\n\t" -#else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "ldr r8, [lr]\n\t" - "ldr r9, [lr, #4]\n\t" - "ldr r10, [lr, #8]\n\t" - "ldr r11, [lr, #12]\n\t" - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" - "ldr r8, [sp, #4]\n\t" - "str r4, [%[out]]\n\t" - "str r5, [%[out], #4]\n\t" - "str r6, [%[out], #8]\n\t" - "str r7, [%[out], #12]\n\t" - "ldm r8, {r4, r5, r6, r7}\n\t" - "subs %[length], %[length], #16\n\t" - "add lr, lr, #16\n\t" - "add %[out], %[out], #16\n\t" - "bne L_AES_GCMSIV_ctr_base_loop_block_192_%=\n\t" - "b L_AES_GCMSIV_ctr_base_end_%=\n\t" - "\n" - "L_AES_GCMSIV_ctr_base_start_block_128_%=:\n\t" - "\n" - "L_AES_GCMSIV_ctr_base_loop_block_128_%=:\n\t" - "push {r1, %[length], lr}\n\t" - "ldr lr, [sp, #16]\n\t" - "rev r8, r4\n\t" - "add r8, r8, #1\n\t" - "rev r8, r8\n\t" - "mov r9, r5\n\t" - "mov r10, r6\n\t" - "mov r11, r7\n\t" - "stm lr, {r8, r9, r10, r11}\n\t" - "ldm %[KS]!, {r8, r9, r10, r11}\n\t" - /* Round: 0 - XOR in key schedule */ - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" - "mov r1, #4\n\t" -#ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE - "bl AES_encrypt_block\n\t" + "vdup.32 q1, r9\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #0x40\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "eor r8, r8, r7\n\t" + "eor r9, r9, r8\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "\n" - "L_AES_GCMSIV_ctr_base_block_nr_128_%=:\n\t" + "strd r4, r5, [%[key]], #8\n\t" +#endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r8, r5, #8\n\t" - "lsr r8, r8, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb r8, r5, ror #16\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r8, r9}\n\t" #else - "ubfx r8, r5, #16, #8\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif - "lsr r11, r4, #24\n\t" + "vdup.32 q1, r9\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #0x80\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r6, #16\n\t" - "lsr lr, lr, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb lr, r6, ror #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx lr, r6, #8, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif + "cmp %[dir], #0\n\t" + "beq L_aes_set_key_arm32_crypto_done_%=\n\t" + "sub %[key], %[key], #0xd0\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0xc0\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0xc0\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0xc0\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0xb0\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0xa0\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0xa0\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0xa0\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0x90\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0x80\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0x80\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0x80\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0x70\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0x60\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0x60\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0x60\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0x50\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0x40\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0x40\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0x40\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #48\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #32\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #32\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #32\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #16\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "aesimc.8 q0, q0\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "b L_aes_set_key_arm32_crypto_done_%=\n\t" + "\n" + "L_aes_set_key_arm32_crypto_start_256_%=:\n\t" + "ldr r4, [%[userKey]], #4\n\t" + "ldr r5, [%[userKey]], #4\n\t" + "ldr r6, [%[userKey]], #4\n\t" + "ldr r7, [%[userKey]], #4\n\t" + "ldr r8, [%[userKey]], #4\n\t" + "ldr r9, [%[userKey]], #4\n\t" + "ldr r10, [%[userKey]], #4\n\t" + "ldr r11, [%[userKey]], #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r7, #24\n\t" - "lsr r2, r2, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb r2, r7\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx r2, r7, #0, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "ldr r8, [r0, r8, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r9, r6, #8\n\t" - "lsr r9, r9, #24\n\t" + "stm r2!, {r8, r9}\n\t" #else - "uxtb r9, r6, ror #16\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r10, r11}\n\t" #else - "ubfx r9, r6, #16, #8\n\t" + "strd r10, r11, [%[key]], #8\n\t" #endif - "eor r8, r8, r11, ror #24\n\t" - "lsr r11, r5, #24\n\t" - "eor r8, r8, lr, ror #8\n\t" + "vdup.32 q1, r11\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #1\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "eor r8, r8, r12\n\t" + "eor r9, r9, r8\n\t" + "eor r10, r10, r9\n\t" + "eor r11, r11, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r7, #16\n\t" - "lsr lr, lr, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb lr, r7, ror #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx lr, r7, #8, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "eor r8, r8, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r4, #24\n\t" - "lsr r2, r2, #24\n\t" + "stm r2!, {r8, r9}\n\t" #else - "uxtb r2, r4\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r10, r11}\n\t" #else - "ubfx r2, r4, #0, #8\n\t" + "strd r10, r11, [%[key]], #8\n\t" #endif - "ldr r9, [r0, r9, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" + "vdup.32 q1, r11\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #2\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "eor r8, r8, r12\n\t" + "eor r9, r9, r8\n\t" + "eor r10, r10, r9\n\t" + "eor r11, r11, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r10, r7, #8\n\t" - "lsr r10, r10, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb r10, r7, ror #16\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx r10, r7, #16, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "eor r9, r9, r11, ror #24\n\t" - "lsr r11, r6, #24\n\t" - "eor r9, r9, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r4, #16\n\t" - "lsr lr, lr, #24\n\t" + "stm r2!, {r8, r9}\n\t" #else - "uxtb lr, r4, ror #8\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r10, r11}\n\t" #else - "ubfx lr, r4, #8, #8\n\t" + "strd r10, r11, [%[key]], #8\n\t" #endif - "eor r9, r9, r2, ror #16\n\t" + "vdup.32 q1, r11\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #4\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "eor r8, r8, r12\n\t" + "eor r9, r9, r8\n\t" + "eor r10, r10, r9\n\t" + "eor r11, r11, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #24\n\t" - "lsr r2, r2, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb r2, r5\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx r2, r5, #0, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "ldr r10, [r0, r10, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r6, #24\n\t" - "lsr r6, r6, #24\n\t" + "stm r2!, {r8, r9}\n\t" #else - "uxtb r6, r6\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r10, r11}\n\t" #else - "ubfx r6, r6, #0, #8\n\t" + "strd r10, r11, [%[key]], #8\n\t" #endif - "eor r10, r10, r11, ror #24\n\t" + "vdup.32 q1, r11\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #8\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "eor r8, r8, r12\n\t" + "eor r9, r9, r8\n\t" + "eor r10, r10, r9\n\t" + "eor r11, r11, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r11, r4, #8\n\t" - "lsr r11, r11, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb r11, r4, ror #16\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx r11, r4, #16, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "eor r10, r10, lr, ror #8\n\t" - "lsr lr, r7, #24\n\t" - "eor r10, r10, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #16\n\t" - "lsr r2, r2, #24\n\t" + "stm r2!, {r8, r9}\n\t" #else - "uxtb r2, r5, ror #8\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r10, r11}\n\t" #else - "ubfx r2, r5, #8, #8\n\t" + "strd r10, r11, [%[key]], #8\n\t" #endif - "ldr r6, [r0, r6, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r6, ror #24\n\t" - "ldm %[KS]!, {r4, r5, r6, r7}\n\t" - "eor r11, r11, lr, ror #24\n\t" - "eor r11, r11, r2, ror #8\n\t" - /* XOR in Key Schedule */ - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" + "vdup.32 q1, r11\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #16\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "eor r8, r8, r12\n\t" + "eor r9, r9, r8\n\t" + "eor r10, r10, r9\n\t" + "eor r11, r11, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r9, #8\n\t" - "lsr r4, r4, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb r4, r9, ror #16\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx r4, r9, #16, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "lsr r7, r8, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r10, #16\n\t" - "lsr lr, lr, #24\n\t" + "stm r2!, {r8, r9}\n\t" #else - "uxtb lr, r10, ror #8\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r10, r11}\n\t" #else - "ubfx lr, r10, #8, #8\n\t" + "strd r10, r11, [%[key]], #8\n\t" #endif + "vdup.32 q1, r11\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #32\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "eor r8, r8, r12\n\t" + "eor r9, r9, r8\n\t" + "eor r10, r10, r9\n\t" + "eor r11, r11, r10\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r11, #24\n\t" - "lsr r2, r2, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb r2, r11\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx r2, r11, #0, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "ldr r4, [r0, r4, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r10, #8\n\t" - "lsr r5, r5, #24\n\t" + "stm r2!, {r8, r9}\n\t" #else - "uxtb r5, r10, ror #16\n\t" + "strd r8, r9, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r10, r11}\n\t" #else - "ubfx r5, r10, #16, #8\n\t" + "strd r10, r11, [%[key]], #8\n\t" #endif - "eor r4, r4, r7, ror #24\n\t" - "lsr r7, r9, #24\n\t" - "eor r4, r4, lr, ror #8\n\t" + "vdup.32 q1, r11\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #0x40\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r11, #16\n\t" - "lsr lr, lr, #24\n\t" + "stm r2!, {r4, r5}\n\t" #else - "uxtb lr, r11, ror #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r6, r7}\n\t" #else - "ubfx lr, r11, #8, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "eor r4, r4, r2, ror #16\n\t" + "cmp %[dir], #0\n\t" + "beq L_aes_set_key_arm32_crypto_done_%=\n\t" + "sub %[key], %[key], #0xf0\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0xe0\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0xe0\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0xe0\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0xd0\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0xc0\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0xc0\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0xc0\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0xb0\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0xa0\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0xa0\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0xa0\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0x90\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0x80\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0x80\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0x80\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0x70\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0x60\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0x60\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0x60\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0x50\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0x40\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0x40\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0x40\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #48\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #32\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #32\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #32\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #16\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "aesimc.8 q0, q0\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "b L_aes_set_key_arm32_crypto_done_%=\n\t" + "\n" + "L_aes_set_key_arm32_crypto_start_128_%=:\n\t" + "ldr r4, [%[userKey]], #4\n\t" + "ldr r5, [%[userKey]], #4\n\t" + "ldr r6, [%[userKey]], #4\n\t" + "ldr r7, [%[userKey]], #4\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r8, #24\n\t" - "lsr r2, r2, #24\n\t" -#else - "uxtb r2, r8\n\t" -#endif + "stm r2!, {r4, r5}\n\t" #else - "ubfx r2, r8, #0, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "ldr r5, [r0, r5, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r11, #8\n\t" - "lsr r6, r6, #24\n\t" -#else - "uxtb r6, r11, ror #16\n\t" -#endif + "stm r2!, {r6, r7}\n\t" #else - "ubfx r6, r11, #16, #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif - "eor r5, r5, r7, ror #24\n\t" - "lsr r7, r10, #24\n\t" - "eor r5, r5, lr, ror #8\n\t" + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #1\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r8, #16\n\t" - "lsr lr, lr, #24\n\t" -#else - "uxtb lr, r8, ror #8\n\t" -#endif + "stm r2!, {r4, r5}\n\t" #else - "ubfx lr, r8, #8, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "eor r5, r5, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r9, #24\n\t" - "lsr r2, r2, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb r2, r9\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #2\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx r2, r9, #0, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "ldr r6, [r0, r6, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r10, r10, #24\n\t" - "lsr r10, r10, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb r10, r10\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #4\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx r10, r10, #0, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "eor r6, r6, r7, ror #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r8, #8\n\t" - "lsr r7, r7, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb r7, r8, ror #16\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #8\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx r7, r8, #16, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "eor r6, r6, lr, ror #8\n\t" - "lsr lr, r11, #24\n\t" - "eor r6, r6, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r9, #16\n\t" - "lsr r2, r2, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb r2, r9, ror #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #16\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx r2, r9, #8, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "ldr r10, [r0, r10, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r7, [r0, r7, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r10, ror #24\n\t" - "ldm %[KS]!, {r8, r9, r10, r11}\n\t" - "eor r7, r7, lr, ror #24\n\t" - "eor r7, r7, r2, ror #8\n\t" - /* XOR in Key Schedule */ - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" - "subs r1, r1, #1\n\t" - "bne L_AES_GCMSIV_ctr_base_block_nr_128_%=\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r8, r5, #8\n\t" - "lsr r8, r8, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb r8, r5, ror #16\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #32\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx r8, r5, #16, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "lsr r11, r4, #24\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r6, #16\n\t" - "lsr lr, lr, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb lr, r6, ror #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #0x40\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx lr, r6, #8, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r7, #24\n\t" - "lsr r2, r2, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb r2, r7\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, #0x80\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx r2, r7, #0, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "ldr r8, [r0, r8, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r9, r6, #8\n\t" - "lsr r9, r9, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb r9, r6, ror #16\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "mov lr, #27\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, lr\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx r9, r6, #16, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "eor r8, r8, r11, ror #24\n\t" - "lsr r11, r5, #24\n\t" - "eor r8, r8, lr, ror #8\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r7, #16\n\t" - "lsr lr, lr, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb lr, r7, ror #8\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif + "vdup.32 q1, r7\n\t" + "vmov.i32 q0, #0\n\t" + "aese.8 q0, q1\n\t" + "vmov.i32 r12, s0\n\t" + "mov lr, #54\n\t" + "ror r12, r12, #8\n\t" + "eor r4, r4, lr\n\t" + "eor r4, r4, r12\n\t" + "eor r5, r5, r4\n\t" + "eor r6, r6, r5\n\t" + "eor r7, r7, r6\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "stm r2!, {r4, r5}\n\t" #else - "ubfx lr, r7, #8, #8\n\t" + "strd r4, r5, [%[key]], #8\n\t" #endif - "eor r8, r8, r2, ror #16\n\t" #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r4, #24\n\t" - "lsr r2, r2, #24\n\t" + "stm r2!, {r6, r7}\n\t" #else - "uxtb r2, r4\n\t" + "strd r6, r7, [%[key]], #8\n\t" #endif + "cmp %[dir], #0\n\t" + "beq L_aes_set_key_arm32_crypto_done_%=\n\t" + "sub %[key], %[key], #0xb0\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0xa0\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0xa0\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0xa0\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0x90\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0x80\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0x80\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0x80\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0x70\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0x60\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0x60\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0x60\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #0x50\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #0x40\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #0x40\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #0x40\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #48\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "add %[key], %[key], #32\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "sub %[key], %[key], #32\n\t" + "aesimc.8 q0, q0\n\t" + "aesimc.8 q1, q1\n\t" + "vst1.32 {q1}, [%[key]]\n\t" + "add %[key], %[key], #32\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "sub %[key], %[key], #16\n\t" + "vld1.32 {q0}, [%[key]]\n\t" + "aesimc.8 q0, q0\n\t" + "vst1.32 {q0}, [%[key]]\n\t" + "\n" + "L_aes_set_key_arm32_crypto_done_%=:\n\t" + "pop {%[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [userKey] "+r" (userKey), [keylen] "+r" (keylen), [key] "+r" (key), + [dir] "+r" (dir), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "ubfx r2, r4, #0, #8\n\t" -#endif - "ldr r9, [r0, r9, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r10, r7, #8\n\t" - "lsr r10, r10, #24\n\t" + : + : [userKey] "r" (userKey), [keylen] "r" (keylen), [key] "r" (key), + [dir] "r" (dir), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "r9", "r10", + "r11", "q0", "q1" + ); +} + +#if defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ + defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || \ + defined(HAVE_AES_CBC) +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_encrypt_AARCH32(const byte* inBlock_p, + byte* outBlock_p, byte* key_p, int nr_p) #else - "uxtb r10, r7, ror #16\n\t" -#endif +WC_OMIT_FRAME_POINTER void AES_encrypt_AARCH32(const byte* inBlock, + byte* outBlock, byte* key, int nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* inBlock __asm__ ("r0") = (const byte*)inBlock_p; + register byte* outBlock __asm__ ("r1") = (byte*)outBlock_p; + register byte* key __asm__ ("r2") = (byte*)key_p; + register int nr __asm__ ("r3") = (int)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r12") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "ubfx r10, r7, #16, #8\n\t" -#endif - "eor r9, r9, r11, ror #24\n\t" - "lsr r11, r6, #24\n\t" - "eor r9, r9, lr, ror #8\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r4, #16\n\t" - "lsr lr, lr, #24\n\t" + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_GCMSIV_ctr_base_te]}\n\t" + "vld1.8 {q0}, [%[inBlock]]\n\t" + "vldm %[key]!, {q1-q4}\n\t" + "aese.8 q0, q1\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q2\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "vldm %[key]!, {q1-q4}\n\t" + "aese.8 q0, q1\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q2\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "subs %[nr], %[nr], #10\n\t" + "vld1.32 {q1-q2}, [%[key]]!\n\t" + "aese.8 q0, q1\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q2\n\t" + "beq L_aes_encrypt_arm32_crypto_round_done_%=\n\t" + "vld1.32 {q1-q2}, [%[key]]!\n\t" + "subs %[nr], %[nr], #2\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q1\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q2\n\t" + "beq L_aes_encrypt_arm32_crypto_round_done_%=\n\t" + "vld1.32 {q1-q2}, [%[key]]!\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q1\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q2\n\t" + "\n" + "L_aes_encrypt_arm32_crypto_round_done_%=:\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "veor.32 q0, q0, q1\n\t" + "vst1.8 {q0}, [%[outBlock]]\n\t" + "pop {%[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [inBlock] "+r" (inBlock), [outBlock] "+r" (outBlock), + [key] "+r" (key), [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : +#else + : + : [inBlock] "r" (inBlock), [outBlock] "r" (outBlock), [key] "r" (key), + [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "q0", "q1", "q2", "q3", "q4" + ); +} + +#endif /* defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || + * defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || + * defined(HAVE_AES_CBC) */ +#if !defined(WC_AES_BITSLICED) || defined(WOLFSSL_AES_DIRECT) || \ + defined(WOLFSSL_AES_COUNTER) +#ifdef HAVE_AES_DECRYPT +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_decrypt_AARCH32(const byte* inBlock_p, + byte* outBlock_p, byte* key_p, int nr_p) +#else +WC_OMIT_FRAME_POINTER void AES_decrypt_AARCH32(const byte* inBlock, + byte* outBlock, byte* key, int nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* inBlock __asm__ ("r0") = (const byte*)inBlock_p; + register byte* outBlock __asm__ ("r1") = (byte*)outBlock_p; + register byte* key __asm__ ("r2") = (byte*)key_p; + register int nr __asm__ ("r3") = (int)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r12") = + (word32*)L_AES_GCMSIV_ctr_base_te; +#else + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_GCMSIV_ctr_base_te]}\n\t" + "vld1.8 {q0}, [%[inBlock]]\n\t" + "vldm %[key]!, {q1-q4}\n\t" + "aesd.8 q0, q1\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q2\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q3\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q4\n\t" + "aesimc.8 q0, q0\n\t" + "vldm %[key]!, {q1-q4}\n\t" + "aesd.8 q0, q1\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q2\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q3\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q4\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q1-q2}, [%[key]]!\n\t" + "aesd.8 q0, q1\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q2\n\t" + "subs %[nr], %[nr], #10\n\t" + "beq L_aes_decrypt_arm32_crypto_round_done_%=\n\t" + "vld1.32 {q1-q2}, [%[key]]!\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q1\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q2\n\t" + "subs %[nr], %[nr], #2\n\t" + "beq L_aes_decrypt_arm32_crypto_round_done_%=\n\t" + "vld1.32 {q1-q2}, [%[key]]!\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q1\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q2\n\t" + "\n" + "L_aes_decrypt_arm32_crypto_round_done_%=:\n\t" + "vld1.32 {q1}, [%[key]]\n\t" + "veor.32 q0, q0, q1\n\t" + "vst1.8 {q0}, [%[outBlock]]\n\t" + "pop {%[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [inBlock] "+r" (inBlock), [outBlock] "+r" (outBlock), + [key] "+r" (key), [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : +#else + : + : [inBlock] "r" (inBlock), [outBlock] "r" (outBlock), [key] "r" (key), + [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "q0", "q1", "q2", "q3", "q4" + ); +} + +#endif /* HAVE_AES_DECRYPT */ +#endif /* !defined(WC_AES_BITSLICED) || defined(WOLFSSL_AES_DIRECT) || + * defined(WOLFSSL_AES_COUNTER) */ +#ifdef HAVE_AES_ECB +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_encrypt_blocks_AARCH32(const byte* in_p, + byte* out_p, word32 sz_p, byte* key_p, int nr_p) #else - "uxtb lr, r4, ror #8\n\t" -#endif +WC_OMIT_FRAME_POINTER void AES_encrypt_blocks_AARCH32(const byte* in, byte* out, + word32 sz, byte* key, int nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* in __asm__ ("r0") = (const byte*)in_p; + register byte* out __asm__ ("r1") = (byte*)out_p; + register word32 sz __asm__ ("r2") = (word32)sz_p; + register byte* key __asm__ ("r3") = (byte*)key_p; + register int nr __asm__ ("r12") = (int)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("lr") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "ubfx lr, r4, #8, #8\n\t" -#endif - "eor r9, r9, r2, ror #16\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #24\n\t" - "lsr r2, r2, #24\n\t" + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" + "ldr r12, [sp]\n\t" + "vldm.32 %[key]!, {q0-q7}\n\t" + "lsr %[sz], %[sz], #4\n\t" + "cmp r12, #12\n\t" + "blt L_aes_encrypt_blocks_arm32_crypto_start_128_%=\n\t" + "bgt L_aes_encrypt_blocks_arm32_crypto_start_256_%=\n\t" + /* AES_ECB_192 */ +#ifndef NO_AES_192 + "vld1.32 {q8-q9}, [%[key]]!\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_encrypt_blocks_arm32_crypto_192_start_1_%=\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_192_start_4_%=:\n\t" + "cmp %[sz], #4\n\t" + "blt L_aes_encrypt_blocks_arm32_crypto_192_start_2_%=\n\t" + "vldm.8 %[in]!, {q12-q15}\n\t" + "aese.8 q12, q0\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q0\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q0\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q1\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q1\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q1\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q2\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q2\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q2\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q3\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q3\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q3\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q4\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q4\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q4\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q5\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q5\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q5\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q6\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q6\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q6\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q7\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q7\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q7\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q8\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q8\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q8\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q9\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q9\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q9\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aese.8 q12, q10\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q10\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q10\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "aese.8 q13, q11\n\t" + "veor.32 q13, q13, q10\n\t" + "aese.8 q14, q11\n\t" + "veor.32 q14, q14, q10\n\t" + "aese.8 q15, q11\n\t" + "veor.32 q15, q15, q10\n\t" + "sub %[key], %[key], #48\n\t" + "sub %[sz], %[sz], #4\n\t" + "vstm.8 %[out]!, {q12-q15}\n\t" + "cmp %[sz], #4\n\t" + "bge L_aes_encrypt_blocks_arm32_crypto_192_start_4_%=\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_192_start_2_%=:\n\t" + "cmp %[sz], #2\n\t" + "blt L_aes_encrypt_blocks_arm32_crypto_192_start_1_%=\n\t" + "vld1.8 {q12-q13}, [%[in]]!\n\t" + "aese.8 q12, q0\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q0\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q1\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q1\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q2\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q2\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q3\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q3\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q4\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q4\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q5\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q5\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q6\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q6\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q7\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q7\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q8\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q8\n\t" + "aesmc.8 q13, q13\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q9\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q9\n\t" + "aesmc.8 q13, q13\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aese.8 q12, q10\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q10\n\t" + "aesmc.8 q13, q13\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "aese.8 q13, q11\n\t" + "veor.32 q13, q13, q10\n\t" + "sub %[key], %[key], #48\n\t" + "sub %[sz], %[sz], #2\n\t" + "vst1.8 {q12-q13}, [%[out]]!\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_192_start_1_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_encrypt_blocks_arm32_crypto_192_done_%=\n\t" + "vld1.8 {q12}, [%[in]]!\n\t" + "aese.8 q12, q0\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q1\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q2\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q3\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q4\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q5\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q6\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q7\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q8\n\t" + "aesmc.8 q12, q12\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q9\n\t" + "aesmc.8 q12, q12\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aese.8 q12, q10\n\t" + "aesmc.8 q12, q12\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "sub %[key], %[key], #48\n\t" + "vst1.8 {q12}, [%[out]]!\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_192_done_%=:\n\t" +#endif /* !NO_AES_192 */ + "b L_aes_encrypt_blocks_arm32_crypto_done_%=\n\t" + /* AES_ECB_256 */ + "\n" + "L_aes_encrypt_blocks_arm32_crypto_start_256_%=:\n\t" +#ifndef NO_AES_256 + "vld1.32 {q8-q9}, [%[key]]!\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_encrypt_blocks_arm32_crypto_256_start_1_%=\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_256_start_4_%=:\n\t" + "cmp %[sz], #4\n\t" + "blt L_aes_encrypt_blocks_arm32_crypto_256_start_2_%=\n\t" + "vldm.8 %[in]!, {q12-q15}\n\t" + "aese.8 q12, q0\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q0\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q0\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q1\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q1\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q1\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q2\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q2\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q2\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q3\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q3\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q3\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q4\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q4\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q4\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q5\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q5\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q5\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q6\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q6\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q6\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q7\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q7\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q7\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q8\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q8\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q8\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q9\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q9\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q9\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aese.8 q12, q10\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q10\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q10\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q11\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q11\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q11\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q11\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aese.8 q12, q10\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q10\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q10\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "aese.8 q13, q11\n\t" + "veor.32 q13, q13, q10\n\t" + "aese.8 q14, q11\n\t" + "veor.32 q14, q14, q10\n\t" + "aese.8 q15, q11\n\t" + "veor.32 q15, q15, q10\n\t" + "sub %[key], %[key], #0x50\n\t" + "sub %[sz], %[sz], #4\n\t" + "vstm.8 %[out]!, {q12-q15}\n\t" + "cmp %[sz], #4\n\t" + "bge L_aes_encrypt_blocks_arm32_crypto_256_start_4_%=\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_256_start_2_%=:\n\t" + "cmp %[sz], #2\n\t" + "blt L_aes_encrypt_blocks_arm32_crypto_256_start_1_%=\n\t" + "vld1.8 {q12-q13}, [%[in]]!\n\t" + "aese.8 q12, q0\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q0\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q1\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q1\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q2\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q2\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q3\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q3\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q4\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q4\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q5\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q5\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q6\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q6\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q7\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q7\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q8\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q8\n\t" + "aesmc.8 q13, q13\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q9\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q9\n\t" + "aesmc.8 q13, q13\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aese.8 q12, q10\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q10\n\t" + "aesmc.8 q13, q13\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q11\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q11\n\t" + "aesmc.8 q13, q13\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aese.8 q12, q10\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q10\n\t" + "aesmc.8 q13, q13\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "aese.8 q13, q11\n\t" + "veor.32 q13, q13, q10\n\t" + "sub %[key], %[key], #0x50\n\t" + "sub %[sz], %[sz], #2\n\t" + "vst1.8 {q12-q13}, [%[out]]!\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_256_start_1_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_encrypt_blocks_arm32_crypto_256_done_%=\n\t" + "vld1.8 {q12}, [%[in]]!\n\t" + "aese.8 q12, q0\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q1\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q2\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q3\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q4\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q5\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q6\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q7\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q8\n\t" + "aesmc.8 q12, q12\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q9\n\t" + "aesmc.8 q12, q12\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aese.8 q12, q10\n\t" + "aesmc.8 q12, q12\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q11\n\t" + "aesmc.8 q12, q12\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aese.8 q12, q10\n\t" + "aesmc.8 q12, q12\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aese.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "sub %[key], %[key], #0x50\n\t" + "vst1.8 {q12}, [%[out]]!\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_256_done_%=:\n\t" +#endif /* !NO_AES_256 */ + "b L_aes_encrypt_blocks_arm32_crypto_done_%=\n\t" + /* AES_ECB_128 */ + "\n" + "L_aes_encrypt_blocks_arm32_crypto_start_128_%=:\n\t" +#ifndef NO_AES_128 + "vldm.32 %[key]!, {q8-q10}\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_encrypt_blocks_arm32_crypto_128_start_1_%=\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_128_start_4_%=:\n\t" + "cmp %[sz], #4\n\t" + "blt L_aes_encrypt_blocks_arm32_crypto_128_start_2_%=\n\t" + "vldm.8 %[in]!, {q12-q15}\n\t" + "aese.8 q12, q0\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q0\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q0\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q1\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q1\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q1\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q2\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q2\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q2\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q3\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q3\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q3\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q4\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q4\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q4\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q5\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q5\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q5\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q6\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q6\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q6\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q7\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q7\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q7\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q8\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q8\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q14, q8\n\t" + "aesmc.8 q14, q14\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q12, q9\n\t" + "veor.32 q12, q12, q10\n\t" + "aese.8 q13, q9\n\t" + "veor.32 q13, q13, q10\n\t" + "aese.8 q14, q9\n\t" + "veor.32 q14, q14, q10\n\t" + "aese.8 q15, q9\n\t" + "veor.32 q15, q15, q10\n\t" + "sub %[sz], %[sz], #4\n\t" + "vstm.8 %[out]!, {q12-q15}\n\t" + "cmp %[sz], #4\n\t" + "bge L_aes_encrypt_blocks_arm32_crypto_128_start_4_%=\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_128_start_2_%=:\n\t" + "cmp %[sz], #2\n\t" + "blt L_aes_encrypt_blocks_arm32_crypto_128_start_1_%=\n\t" + "vld1.8 {q12-q13}, [%[in]]!\n\t" + "aese.8 q12, q0\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q0\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q1\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q1\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q2\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q2\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q3\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q3\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q4\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q4\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q5\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q5\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q6\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q6\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q7\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q7\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q8\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q13, q8\n\t" + "aesmc.8 q13, q13\n\t" + "aese.8 q12, q9\n\t" + "veor.32 q12, q12, q10\n\t" + "aese.8 q13, q9\n\t" + "veor.32 q13, q13, q10\n\t" + "sub %[sz], %[sz], #2\n\t" + "vst1.8 {q12-q13}, [%[out]]!\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_128_start_1_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_encrypt_blocks_arm32_crypto_128_done_%=\n\t" + "vld1.8 {q12}, [%[in]]!\n\t" + "aese.8 q12, q0\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q1\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q2\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q3\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q4\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q5\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q6\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q7\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q8\n\t" + "aesmc.8 q12, q12\n\t" + "aese.8 q12, q9\n\t" + "veor.32 q12, q12, q10\n\t" + "vst1.8 {q12}, [%[out]]!\n\t" + "\n" + "L_aes_encrypt_blocks_arm32_crypto_128_done_%=:\n\t" +#endif /* !NO_AES_128 */ + "\n" + "L_aes_encrypt_blocks_arm32_crypto_done_%=:\n\t" + "pop {%[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), + [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "uxtb r2, r5\n\t" -#endif + : + : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [key] "r" (key), + [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", + "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#ifdef HAVE_AES_DECRYPT +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_decrypt_blocks_AARCH32(const byte* in_p, + byte* out_p, word32 sz_p, byte* key_p, int nr_p) #else - "ubfx r2, r5, #0, #8\n\t" -#endif - "ldr r10, [r0, r10, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r6, #24\n\t" - "lsr r6, r6, #24\n\t" +WC_OMIT_FRAME_POINTER void AES_decrypt_blocks_AARCH32(const byte* in, byte* out, + word32 sz, byte* key, int nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* in __asm__ ("r0") = (const byte*)in_p; + register byte* out __asm__ ("r1") = (byte*)out_p; + register word32 sz __asm__ ("r2") = (word32)sz_p; + register byte* key __asm__ ("r3") = (byte*)key_p; + register int nr __asm__ ("r12") = (int)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("lr") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "uxtb r6, r6\n\t" -#endif + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" + "ldr r12, [sp]\n\t" + "vldm.32 %[key]!, {q0-q7}\n\t" + "lsr %[sz], %[sz], #4\n\t" + "cmp r12, #12\n\t" + "blt L_aes_decrypt_blocks_arm32_crypto_start_128_%=\n\t" + "bgt L_aes_decrypt_blocks_arm32_crypto_start_256_%=\n\t" + /* AES_ECB_192 */ +#ifndef NO_AES_192 + "vld1.32 {q8-q9}, [%[key]]!\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_decrypt_blocks_arm32_crypto_192_start_1_%=\n\t" + "cmp %[sz], #4\n\t" + "blt L_aes_decrypt_blocks_arm32_crypto_192_start_2_%=\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_192_start_4_%=:\n\t" + "vldm.8 %[in]!, {q12-q15}\n\t" + "aesd.8 q12, q0\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q0\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q0\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q0\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q1\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q1\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q1\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q1\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q2\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q2\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q2\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q2\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q3\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q3\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q3\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q3\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q4\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q4\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q4\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q4\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q5\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q5\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q5\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q5\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q6\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q6\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q6\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q6\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q7\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q7\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q7\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q7\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q8\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q8\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q8\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q8\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q9\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q9\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q9\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q9\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aesd.8 q12, q10\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q10\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q10\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q10\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "aesd.8 q13, q11\n\t" + "veor.32 q13, q13, q10\n\t" + "aesd.8 q14, q11\n\t" + "veor.32 q14, q14, q10\n\t" + "aesd.8 q15, q11\n\t" + "veor.32 q15, q15, q10\n\t" + "sub %[key], %[key], #48\n\t" + "sub %[sz], %[sz], #4\n\t" + "vstm.8 %[out]!, {q12-q15}\n\t" + "cmp %[sz], #4\n\t" + "bge L_aes_decrypt_blocks_arm32_crypto_192_start_4_%=\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_192_start_2_%=:\n\t" + "cmp %[sz], #2\n\t" + "blt L_aes_decrypt_blocks_arm32_crypto_192_start_1_%=\n\t" + "vld1.8 {q12-q13}, [%[in]]!\n\t" + "aesd.8 q12, q0\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q0\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q1\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q1\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q2\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q2\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q3\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q3\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q4\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q4\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q5\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q5\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q6\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q6\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q7\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q7\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q8\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q8\n\t" + "aesimc.8 q13, q13\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q9\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q9\n\t" + "aesimc.8 q13, q13\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aesd.8 q12, q10\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q10\n\t" + "aesimc.8 q13, q13\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "aesd.8 q13, q11\n\t" + "veor.32 q13, q13, q10\n\t" + "sub %[key], %[key], #48\n\t" + "sub %[sz], %[sz], #2\n\t" + "vst1.8 {q12-q13}, [%[out]]!\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_192_start_1_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_decrypt_blocks_arm32_crypto_192_done_%=\n\t" + "vld1.8 {q12}, [%[in]]!\n\t" + "aesd.8 q12, q0\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q1\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q2\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q3\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q4\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q5\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q6\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q7\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q8\n\t" + "aesimc.8 q12, q12\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q9\n\t" + "aesimc.8 q12, q12\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aesd.8 q12, q10\n\t" + "aesimc.8 q12, q12\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "sub %[key], %[key], #48\n\t" + "vst1.8 {q12}, [%[out]]!\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_192_done_%=:\n\t" +#endif /* !NO_AES_192 */ + "b L_aes_decrypt_blocks_arm32_crypto_done_%=\n\t" + /* AES_ECB_256 */ + "\n" + "L_aes_decrypt_blocks_arm32_crypto_start_256_%=:\n\t" +#ifndef NO_AES_256 + "vld1.32 {q8-q9}, [%[key]]!\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_decrypt_blocks_arm32_crypto_256_start_1_%=\n\t" + "cmp %[sz], #4\n\t" + "blt L_aes_decrypt_blocks_arm32_crypto_256_start_2_%=\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_256_start_4_%=:\n\t" + "vldm.8 %[in]!, {q12-q15}\n\t" + "aesd.8 q12, q0\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q0\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q0\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q0\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q1\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q1\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q1\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q1\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q2\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q2\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q2\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q2\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q3\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q3\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q3\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q3\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q4\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q4\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q4\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q4\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q5\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q5\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q5\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q5\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q6\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q6\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q6\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q6\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q7\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q7\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q7\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q7\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q8\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q8\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q8\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q8\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q9\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q9\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q9\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q9\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aesd.8 q12, q10\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q10\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q10\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q10\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q11\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q11\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q11\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q11\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aesd.8 q12, q10\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q10\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q10\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q10\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "aesd.8 q13, q11\n\t" + "veor.32 q13, q13, q10\n\t" + "aesd.8 q14, q11\n\t" + "veor.32 q14, q14, q10\n\t" + "aesd.8 q15, q11\n\t" + "veor.32 q15, q15, q10\n\t" + "sub %[key], %[key], #0x50\n\t" + "sub %[sz], %[sz], #4\n\t" + "vstm.8 %[out]!, {q12-q15}\n\t" + "cmp %[sz], #4\n\t" + "bge L_aes_decrypt_blocks_arm32_crypto_256_start_4_%=\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_256_start_2_%=:\n\t" + "cmp %[sz], #2\n\t" + "blt L_aes_decrypt_blocks_arm32_crypto_256_start_1_%=\n\t" + "vld1.8 {q12-q13}, [%[in]]!\n\t" + "aesd.8 q12, q0\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q0\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q1\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q1\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q2\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q2\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q3\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q3\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q4\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q4\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q5\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q5\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q6\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q6\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q7\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q7\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q8\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q8\n\t" + "aesimc.8 q13, q13\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q9\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q9\n\t" + "aesimc.8 q13, q13\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aesd.8 q12, q10\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q10\n\t" + "aesimc.8 q13, q13\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q11\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q11\n\t" + "aesimc.8 q13, q13\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aesd.8 q12, q10\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q10\n\t" + "aesimc.8 q13, q13\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "aesd.8 q13, q11\n\t" + "veor.32 q13, q13, q10\n\t" + "sub %[key], %[key], #0x50\n\t" + "sub %[sz], %[sz], #2\n\t" + "vst1.8 {q12-q13}, [%[out]]!\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_256_start_1_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_decrypt_blocks_arm32_crypto_256_done_%=\n\t" + "vld1.8 {q12}, [%[in]]!\n\t" + "aesd.8 q12, q0\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q1\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q2\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q3\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q4\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q5\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q6\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q7\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q8\n\t" + "aesimc.8 q12, q12\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q9\n\t" + "aesimc.8 q12, q12\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aesd.8 q12, q10\n\t" + "aesimc.8 q12, q12\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q11\n\t" + "aesimc.8 q12, q12\n\t" + "vld1.32 {q11}, [%[key]]!\n\t" + "aesd.8 q12, q10\n\t" + "aesimc.8 q12, q12\n\t" + "vld1.32 {q10}, [%[key]]!\n\t" + "aesd.8 q12, q11\n\t" + "veor.32 q12, q12, q10\n\t" + "sub %[key], %[key], #0x50\n\t" + "vst1.8 {q12}, [%[out]]!\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_256_done_%=:\n\t" +#endif /* !NO_AES_256 */ + "b L_aes_decrypt_blocks_arm32_crypto_done_%=\n\t" + /* AES_ECB_128 */ + "\n" + "L_aes_decrypt_blocks_arm32_crypto_start_128_%=:\n\t" +#ifndef NO_AES_128 + "vldm.32 %[key]!, {q8-q10}\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_decrypt_blocks_arm32_crypto_128_start_1_%=\n\t" + "cmp %[sz], #4\n\t" + "blt L_aes_decrypt_blocks_arm32_crypto_128_start_2_%=\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_128_start_4_%=:\n\t" + "vldm.8 %[in]!, {q12-q15}\n\t" + "aesd.8 q12, q0\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q0\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q0\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q0\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q1\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q1\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q1\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q1\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q2\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q2\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q2\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q2\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q3\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q3\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q3\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q3\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q4\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q4\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q4\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q4\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q5\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q5\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q5\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q5\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q6\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q6\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q6\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q6\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q7\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q7\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q7\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q7\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q8\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q8\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q14, q8\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q8\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q12, q9\n\t" + "veor.32 q12, q12, q10\n\t" + "aesd.8 q13, q9\n\t" + "veor.32 q13, q13, q10\n\t" + "aesd.8 q14, q9\n\t" + "veor.32 q14, q14, q10\n\t" + "aesd.8 q15, q9\n\t" + "veor.32 q15, q15, q10\n\t" + "sub %[sz], %[sz], #4\n\t" + "vstm.8 %[out]!, {q12-q15}\n\t" + "cmp %[sz], #4\n\t" + "bge L_aes_decrypt_blocks_arm32_crypto_128_start_4_%=\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_128_start_2_%=:\n\t" + "cmp %[sz], #2\n\t" + "blt L_aes_decrypt_blocks_arm32_crypto_128_start_1_%=\n\t" + "vld1.8 {q12-q13}, [%[in]]!\n\t" + "aesd.8 q12, q0\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q0\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q1\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q1\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q2\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q2\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q3\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q3\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q4\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q4\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q5\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q5\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q6\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q6\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q7\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q7\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q8\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q13, q8\n\t" + "aesimc.8 q13, q13\n\t" + "aesd.8 q12, q9\n\t" + "veor.32 q12, q12, q10\n\t" + "aesd.8 q13, q9\n\t" + "veor.32 q13, q13, q10\n\t" + "sub %[sz], %[sz], #2\n\t" + "vst1.8 {q12-q13}, [%[out]]!\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_128_start_1_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_decrypt_blocks_arm32_crypto_128_done_%=\n\t" + "vld1.8 {q12}, [%[in]]!\n\t" + "aesd.8 q12, q0\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q1\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q2\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q3\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q4\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q5\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q6\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q7\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q8\n\t" + "aesimc.8 q12, q12\n\t" + "aesd.8 q12, q9\n\t" + "veor.32 q12, q12, q10\n\t" + "vst1.8 {q12}, [%[out]]!\n\t" + "\n" + "L_aes_decrypt_blocks_arm32_crypto_128_done_%=:\n\t" +#endif /* !NO_AES_128 */ + "\n" + "L_aes_decrypt_blocks_arm32_crypto_done_%=:\n\t" + "pop {%[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), + [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "ubfx r6, r6, #0, #8\n\t" -#endif - "eor r10, r10, r11, ror #24\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r11, r4, #8\n\t" - "lsr r11, r11, #24\n\t" + : + : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [key] "r" (key), + [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", + "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#endif /* HAVE_AES_DECRYPT */ +#endif /* HAVE_AES_ECB */ +#ifdef HAVE_AES_CBC +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_CBC_encrypt_AARCH32(const byte* in_p, + byte* out_p, word32 sz_p, byte* reg_p, byte* key_p, int nr_p) #else - "uxtb r11, r4, ror #16\n\t" -#endif +WC_OMIT_FRAME_POINTER void AES_CBC_encrypt_AARCH32(const byte* in, byte* out, + word32 sz, byte* reg, byte* key, int nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* in __asm__ ("r0") = (const byte*)in_p; + register byte* out __asm__ ("r1") = (byte*)out_p; + register word32 sz __asm__ ("r2") = (word32)sz_p; + register byte* reg __asm__ ("r3") = (byte*)reg_p; + register byte* key __asm__ ("r12") = (byte*)key_p; + register int nr __asm__ ("lr") = (int)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r4") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "ubfx r11, r4, #16, #8\n\t" -#endif - "eor r10, r10, lr, ror #8\n\t" - "lsr lr, r7, #24\n\t" - "eor r10, r10, r2, ror #16\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r5, #16\n\t" - "lsr r2, r2, #24\n\t" + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_GCMSIV_ctr_base_te]}\n\t" + "push {%[key], %[nr]}\n\t" + "ldr r12, [sp]\n\t" + "ldr lr, [sp, #4]\n\t" + "vldm.32 r12!, {q0-q7}\n\t" + "vld1.32 {q15}, [%[reg]]\n\t" + "subs lr, lr, #12\n\t" + "lsr %[sz], %[sz], #4\n\t" + "blt L_aes_cbc_encrypt_arm32_crypto_start_128_%=\n\t" + "bgt L_aes_cbc_encrypt_arm32_crypto_start_256_%=\n\t" + /* AES_CBC_192 */ +#ifndef NO_AES_192 + "vld1.8 {q14}, [%[in]]!\n\t" + "vldm.32 r12!, {q8-q12}\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_cbc_encrypt_arm32_crypto_192_start_1_%=\n\t" + "cmp %[sz], #4\n\t" + "blt L_aes_cbc_encrypt_arm32_crypto_192_start_2_%=\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_192_start_4_%=:\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q11\n\t" + "veor.32 q15, q15, q12\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q11\n\t" + "veor.32 q15, q15, q12\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q11\n\t" + "veor.32 q15, q15, q12\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "subs %[sz], %[sz], #4\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q11\n\t" + "veor.32 q15, q15, q12\n\t" + "beq L_aes_cbc_encrypt_arm32_crypto_192_done_%=\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "cmp %[sz], #4\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "bge L_aes_cbc_encrypt_arm32_crypto_192_start_4_%=\n\t" + "cmp %[sz], #2\n\t" + "blt L_aes_cbc_encrypt_arm32_crypto_192_start_1_%=\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_192_start_2_%=:\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q11\n\t" + "veor.32 q15, q15, q12\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "subs %[sz], %[sz], #2\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q11\n\t" + "veor.32 q15, q15, q12\n\t" + "beq L_aes_cbc_encrypt_arm32_crypto_192_done_%=\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_192_start_1_%=:\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q11\n\t" + "veor.32 q15, q15, q12\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_192_done_%=:\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" +#endif /* !NO_AES_192 */ + "b L_aes_cbc_encrypt_arm32_crypto_done_%=\n\t" + /* AES_CBC_256 */ + "\n" + "L_aes_cbc_encrypt_arm32_crypto_start_256_%=:\n\t" +#ifndef NO_AES_256 + "vld1.8 {q14}, [%[in]]!\n\t" + "vldm.32 r12!, {q8-q11}\n\t" + "add r12, r12, #16\n\t" + "vld1.32 {q12-q13}, [r12]\n\t" + "sub r12, r12, #16\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_cbc_encrypt_arm32_crypto_256_start_1_%=\n\t" + "cmp %[sz], #4\n\t" + "blt L_aes_cbc_encrypt_arm32_crypto_256_start_2_%=\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_256_start_4_%=:\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q14}, [r12]\n\t" + "aese.8 q15, q11\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q14\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q12\n\t" + "veor.32 q15, q15, q13\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q14}, [r12]\n\t" + "aese.8 q15, q11\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q14\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q12\n\t" + "veor.32 q15, q15, q13\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q14}, [r12]\n\t" + "aese.8 q15, q11\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q14\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q12\n\t" + "veor.32 q15, q15, q13\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q14}, [r12]\n\t" + "aese.8 q15, q11\n\t" + "aesmc.8 q15, q15\n\t" + "subs %[sz], %[sz], #4\n\t" + "aese.8 q15, q14\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q12\n\t" + "veor.32 q15, q15, q13\n\t" + "beq L_aes_cbc_encrypt_arm32_crypto_256_done_%=\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "cmp %[sz], #4\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "bge L_aes_cbc_encrypt_arm32_crypto_256_start_4_%=\n\t" + "cmp %[sz], #2\n\t" + "blt L_aes_cbc_encrypt_arm32_crypto_256_start_1_%=\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_256_start_2_%=:\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q14}, [r12]\n\t" + "aese.8 q15, q11\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q14\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q12\n\t" + "veor.32 q15, q15, q13\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q14}, [r12]\n\t" + "aese.8 q15, q11\n\t" + "aesmc.8 q15, q15\n\t" + "subs %[sz], %[sz], #2\n\t" + "aese.8 q15, q14\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q12\n\t" + "veor.32 q15, q15, q13\n\t" + "beq L_aes_cbc_encrypt_arm32_crypto_256_done_%=\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_256_start_1_%=:\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q10\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.32 {q14}, [r12]\n\t" + "aese.8 q15, q11\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q14\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q12\n\t" + "veor.32 q15, q15, q13\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_256_done_%=:\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" +#endif /* !NO_AES_256 */ + "b L_aes_cbc_encrypt_arm32_crypto_done_%=\n\t" + /* AES_CBC_128 */ + "\n" + "L_aes_cbc_encrypt_arm32_crypto_start_128_%=:\n\t" +#ifndef NO_AES_128 + "vld1.8 {q14}, [%[in]]!\n\t" + "vldm.32 r12!, {q8-q10}\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_cbc_encrypt_arm32_crypto_128_start_1_%=\n\t" + "cmp %[sz], #4\n\t" + "blt L_aes_cbc_encrypt_arm32_crypto_128_start_2_%=\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_128_start_4_%=:\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q9\n\t" + "veor.32 q15, q15, q10\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q9\n\t" + "veor.32 q15, q15, q10\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q9\n\t" + "veor.32 q15, q15, q10\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "subs %[sz], %[sz], #4\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "veor.32 q15, q15, q10\n\t" + "beq L_aes_cbc_encrypt_arm32_crypto_128_done_%=\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "cmp %[sz], #4\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "bge L_aes_cbc_encrypt_arm32_crypto_128_start_4_%=\n\t" + "cmp %[sz], #2\n\t" + "blt L_aes_cbc_encrypt_arm32_crypto_128_start_1_%=\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_128_start_2_%=:\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q15, q9\n\t" + "veor.32 q15, q15, q10\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "subs %[sz], %[sz], #2\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "veor.32 q15, q15, q10\n\t" + "beq L_aes_cbc_encrypt_arm32_crypto_128_done_%=\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_128_start_1_%=:\n\t" + "veor.32 q15, q15, q14\n\t" + "aese.8 q15, q0\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q1\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q2\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q3\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q4\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q5\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q6\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q7\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q8\n\t" + "aesmc.8 q15, q15\n\t" + "aese.8 q15, q9\n\t" + "veor.32 q15, q15, q10\n\t" + "\n" + "L_aes_cbc_encrypt_arm32_crypto_128_done_%=:\n\t" + "vst1.8 {q15}, [%[out]]!\n\t" +#endif /* !NO_AES_128 */ + "\n" + "L_aes_cbc_encrypt_arm32_crypto_done_%=:\n\t" + "vst1.32 {q15}, [%[reg]]\n\t" + "pop {%[key], %[nr]}\n\t" + "pop {%[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [reg] "+r" (reg), + [key] "+r" (key), [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "uxtb r2, r5, ror #8\n\t" -#endif + : + : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [reg] "r" (reg), + [key] "r" (key), [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", + "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#ifdef HAVE_AES_DECRYPT +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_CBC_decrypt_AARCH32(const byte* in_p, + byte* out_p, word32 sz_p, byte* reg_p, byte* key_p, int nr_p) #else - "ubfx r2, r5, #8, #8\n\t" -#endif - "ldr r6, [r0, r6, lsl #2]\n\t" - "ldr lr, [r0, lr, lsl #2]\n\t" - "ldr r11, [r0, r11, lsl #2]\n\t" - "ldr r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r6, ror #24\n\t" - "ldm %[KS]!, {r4, r5, r6, r7}\n\t" - "eor r11, r11, lr, ror #24\n\t" - "eor r11, r11, r2, ror #8\n\t" - /* XOR in Key Schedule */ - "eor r8, r8, r4\n\t" - "eor r9, r9, r5\n\t" - "eor r10, r10, r6\n\t" - "eor r11, r11, r7\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r4, r11, #24\n\t" - "lsr r4, r4, #24\n\t" +WC_OMIT_FRAME_POINTER void AES_CBC_decrypt_AARCH32(const byte* in, byte* out, + word32 sz, byte* reg, byte* key, int nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* in __asm__ ("r0") = (const byte*)in_p; + register byte* out __asm__ ("r1") = (byte*)out_p; + register word32 sz __asm__ ("r2") = (word32)sz_p; + register byte* reg __asm__ ("r3") = (byte*)reg_p; + register byte* key __asm__ ("r12") = (byte*)key_p; + register int nr __asm__ ("lr") = (int)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r4") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "uxtb r4, r11\n\t" -#endif + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_GCMSIV_ctr_base_te]}\n\t" + "push {%[key], %[nr]}\n\t" + "ldr r12, [sp]\n\t" + "ldr lr, [sp, #4]\n\t" + "vldm.32 r12!, {q0-q7}\n\t" + "vld1.32 {q13}, [%[reg]]\n\t" + "lsr %[sz], %[sz], #4\n\t" + "cmp lr, #12\n\t" + "blt L_aes_cbc_decrypt_blocks_arm32_crypto_start_128_%=\n\t" + "bgt L_aes_cbc_decrypt_blocks_arm32_crypto_start_256_%=\n\t" + /* AES_CBC_192 */ +#ifndef NO_AES_192 + "vld1.32 {q8}, [r12]!\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_1_%=\n\t" + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_2_%=:\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "vmov q11, q13\n\t" + "vmov q12, q14\n\t" + "vmov q13, q15\n\t" + "aesd.8 q14, q0\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q0\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q1\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q1\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q2\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q2\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q3\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q3\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q4\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q4\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q5\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q5\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q6\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q6\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q7\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q7\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q9}, [r12]!\n\t" + "aesd.8 q14, q8\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q8\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q10}, [r12]!\n\t" + "aesd.8 q14, q9\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q9\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q9}, [r12]!\n\t" + "aesd.8 q14, q10\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q10\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q10}, [r12]\n\t" + "aesd.8 q14, q9\n\t" + "aesd.8 q15, q9\n\t" + "sub %[sz], %[sz], #2\n\t" + "veor.32 q14, q14, q10\n\t" + "veor.32 q15, q15, q10\n\t" + "cmp %[sz], #1\n\t" + "veor.32 q14, q14, q11\n\t" + "veor.32 q15, q15, q12\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "sub r12, r12, #48\n\t" + "blt L_aes_cbc_decrypt_blocks_arm32_crypto_192_done_%=\n\t" + "bgt L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_2_%=\n\t" + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_192_start_1_%=:\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "vmov q11, q13\n\t" + "vmov q13, q14\n\t" + "aesd.8 q14, q0\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q1\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q2\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q3\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q4\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q5\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q6\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q7\n\t" + "aesimc.8 q14, q14\n\t" + "vld1.32 {q9}, [r12]!\n\t" + "aesd.8 q14, q8\n\t" + "aesimc.8 q14, q14\n\t" + "vld1.32 {q10}, [r12]!\n\t" + "aesd.8 q14, q9\n\t" + "aesimc.8 q14, q14\n\t" + "vld1.32 {q9}, [r12]!\n\t" + "aesd.8 q14, q10\n\t" + "aesimc.8 q14, q14\n\t" + "vld1.32 {q10}, [r12]\n\t" + "aesd.8 q14, q9\n\t" + "veor.32 q14, q14, q10\n\t" + "veor.32 q14, q14, q11\n\t" + "vst1.8 {q14}, [%[out]]!\n\t" + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_192_done_%=:\n\t" +#endif /* !NO_AES_192 */ + "b L_aes_cbc_decrypt_blocks_arm32_crypto_done_%=\n\t" + /* AES_CBC_256 */ + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_start_256_%=:\n\t" +#ifndef NO_AES_256 + "vld1.32 {q8}, [r12]!\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_1_%=\n\t" + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_2_%=:\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "vmov q11, q13\n\t" + "vmov q12, q14\n\t" + "vmov q13, q15\n\t" + "aesd.8 q14, q0\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q0\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q1\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q1\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q2\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q2\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q3\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q3\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q4\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q4\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q5\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q5\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q6\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q6\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q7\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q7\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q9}, [r12]!\n\t" + "aesd.8 q14, q8\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q8\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q10}, [r12]!\n\t" + "aesd.8 q14, q9\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q9\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q9}, [r12]!\n\t" + "aesd.8 q14, q10\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q10\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q10}, [r12]!\n\t" + "aesd.8 q14, q9\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q9\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q9}, [r12]!\n\t" + "aesd.8 q14, q10\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q10\n\t" + "aesimc.8 q15, q15\n\t" + "vld1.32 {q10}, [r12]\n\t" + "aesd.8 q14, q9\n\t" + "aesd.8 q15, q9\n\t" + "sub %[sz], %[sz], #2\n\t" + "veor.32 q14, q14, q10\n\t" + "veor.32 q15, q15, q10\n\t" + "cmp %[sz], #1\n\t" + "veor.32 q14, q14, q11\n\t" + "veor.32 q15, q15, q12\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "sub r12, r12, #0x50\n\t" + "blt L_aes_cbc_decrypt_blocks_arm32_crypto_256_done_%=\n\t" + "bgt L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_2_%=\n\t" + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_256_start_1_%=:\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "vmov q11, q13\n\t" + "vmov q13, q14\n\t" + "aesd.8 q14, q0\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q1\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q2\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q3\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q4\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q5\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q6\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q7\n\t" + "aesimc.8 q14, q14\n\t" + "vld1.32 {q9}, [r12]!\n\t" + "aesd.8 q14, q8\n\t" + "aesimc.8 q14, q14\n\t" + "vld1.32 {q10}, [r12]!\n\t" + "aesd.8 q14, q9\n\t" + "aesimc.8 q14, q14\n\t" + "vld1.32 {q9}, [r12]!\n\t" + "aesd.8 q14, q10\n\t" + "aesimc.8 q14, q14\n\t" + "vld1.32 {q10}, [r12]!\n\t" + "aesd.8 q14, q9\n\t" + "aesimc.8 q14, q14\n\t" + "vld1.32 {q9}, [r12]!\n\t" + "aesd.8 q14, q10\n\t" + "aesimc.8 q14, q14\n\t" + "vld1.32 {q10}, [r12]\n\t" + "aesd.8 q14, q9\n\t" + "veor.32 q14, q14, q10\n\t" + "veor.32 q14, q14, q11\n\t" + "vst1.8 {q14}, [%[out]]!\n\t" + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_256_done_%=:\n\t" +#endif /* !NO_AES_256 */ + "b L_aes_cbc_decrypt_blocks_arm32_crypto_done_%=\n\t" + /* AES_CBC_128 */ + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_start_128_%=:\n\t" +#ifndef NO_AES_128 + "vldm.32 r12!, {q8-q10}\n\t" + "cmp %[sz], #1\n\t" + "beq L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_1_%=\n\t" + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_2_%=:\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "vmov q11, q13\n\t" + "vmov q12, q14\n\t" + "vmov q13, q15\n\t" + "aesd.8 q14, q0\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q0\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q1\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q1\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q2\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q2\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q3\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q3\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q4\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q4\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q5\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q5\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q6\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q6\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q7\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q7\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q8\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q15, q8\n\t" + "aesimc.8 q15, q15\n\t" + "aesd.8 q14, q9\n\t" + "aesd.8 q15, q9\n\t" + "sub %[sz], %[sz], #2\n\t" + "veor.32 q14, q14, q10\n\t" + "veor.32 q15, q15, q10\n\t" + "cmp %[sz], #1\n\t" + "veor.32 q14, q14, q11\n\t" + "veor.32 q15, q15, q12\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "blt L_aes_cbc_decrypt_blocks_arm32_crypto_128_done_%=\n\t" + "bgt L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_2_%=\n\t" + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_128_start_1_%=:\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "vmov q11, q13\n\t" + "vmov q13, q14\n\t" + "aesd.8 q14, q0\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q1\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q2\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q3\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q4\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q5\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q6\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q7\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q8\n\t" + "aesimc.8 q14, q14\n\t" + "aesd.8 q14, q9\n\t" + "veor.32 q14, q14, q10\n\t" + "veor.32 q14, q14, q11\n\t" + "vst1.8 {q14}, [%[out]]!\n\t" + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_128_done_%=:\n\t" +#endif /* !NO_AES_128 */ + "\n" + "L_aes_cbc_decrypt_blocks_arm32_crypto_done_%=:\n\t" + "vst1.32 {q13}, [%[reg]]\n\t" + "pop {%[key], %[nr]}\n\t" + "pop {%[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [reg] "+r" (reg), + [key] "+r" (key), [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "ubfx r4, r11, #0, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r10, #16\n\t" - "lsr r7, r7, #24\n\t" + : + : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [reg] "r" (reg), + [key] "r" (key), [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", + "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#endif /* HAVE_AES_DECRYPT */ +#endif /* HAVE_AES_CBC */ +#ifdef WOLFSSL_AES_COUNTER +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_CTR_encrypt_AARCH32(const byte* in_p, + byte* out_p, word32 sz_p, byte* reg_p, byte* key_p, byte* tmp_p, + word32* left_p, word32 nr_p) #else - "uxtb r7, r10, ror #8\n\t" -#endif +WC_OMIT_FRAME_POINTER void AES_CTR_encrypt_AARCH32(const byte* in, byte* out, + word32 sz, byte* reg, byte* key, byte* tmp, word32* left, word32 nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* in __asm__ ("r0") = (const byte*)in_p; + register byte* out __asm__ ("r1") = (byte*)out_p; + register word32 sz __asm__ ("r2") = (word32)sz_p; + register byte* reg __asm__ ("r3") = (byte*)reg_p; + register byte* key __asm__ ("r12") = (byte*)key_p; + register byte* tmp __asm__ ("lr") = (byte*)tmp_p; + register word32* left __asm__ ("r4") = (word32*)left_p; + register word32 nr __asm__ ("r5") = (word32)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r6") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "ubfx r7, r10, #8, #8\n\t" -#endif -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r9, #8\n\t" - "lsr lr, lr, #24\n\t" + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[left], %[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" + "push {%[key], %[tmp]}\n\t" + "vld1.32 {q0}, [%[reg]]\n\t" + "ldr r12, [sp]\n\t" + "vrev32.8 q2, q0\n\t" + "lsr r4, %[sz], #4\n\t" + "vmov r5, r6, d4\n\t" + "and %[sz], %[sz], #15\n\t" + "vmov r7, r8, d5\n\t" + "vldm.32 r12!, {q3-q10}\n\t" + "ldr lr, [sp, #12]\n\t" + "cmp lr, #12\n\t" + "blt L_aes_ctr_encrypt_arm32_crypto_start_128_%=\n\t" + "bgt L_aes_ctr_encrypt_arm32_crypto_start_256_%=\n\t" + /* AES_CTR_192 */ +#ifndef NO_AES_192 + "vldm.32 r12!, {q11-q13}\n\t" + "mov lr, #1\n\t" + "cmp r4, #1\n\t" + "blt L_aes_ctr_encrypt_arm32_crypto_192_done_%=\n\t" + "beq L_aes_ctr_encrypt_arm32_crypto_192_start_1_%=\n\t" + "adds r8, r8, #1\n\t" + "adcs r7, r7, #0\n\t" + "adcs r6, r6, #0\n\t" + "adc r5, r5, #0\n\t" + "vmov d3, r7, r8\n\t" + "vmov d2, r5, r6\n\t" + "vrev32.8 q1, q1\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_192_start_2_%=:\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "adds r8, r8, #1\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "adcs r7, r7, #0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "adcs r6, r6, #0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "adc r5, r5, #0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q0, q11\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q11\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q0, q12\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q12\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "vmov d5, r7, r8\n\t" + "aese.8 q0, q13\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q13\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q15}, [r12]\n\t" + "vmov d4, r5, r6\n\t" + "aese.8 q0, q14\n\t" + "aese.8 q1, q14\n\t" + "sub r4, r4, #2\n\t" + "veor.32 q0, q0, q15\n\t" + "veor.32 q1, q1, q15\n\t" + "adds r8, r8, #1\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "adcs r7, r7, #0\n\t" + "sub r12, r12, #16\n\t" + "veor.32 q14, q14, q0\n\t" + "veor.32 q15, q15, q1\n\t" + "adcs r6, r6, #0\n\t" + "vrev32.8 q0, q2\n\t" + "adc r5, r5, #0\n\t" + "vmov d2, r5, r6\n\t" + "vmov d3, r7, r8\n\t" + "cmp r4, #1\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "vrev32.8 q1, q1\n\t" + "bgt L_aes_ctr_encrypt_arm32_crypto_192_start_2_%=\n\t" + "mov lr, #0\n\t" + "blt L_aes_ctr_encrypt_arm32_crypto_192_done_%=\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_192_start_1_%=:\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "adds r8, r8, lr\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r7, r7, #0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r6, r6, #0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "adc r5, r5, #0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d5, r7, r8\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d4, r5, r6\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q11\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q12\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aese.8 q0, q13\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q15}, [r12]\n\t" + "aese.8 q0, q14\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "veor.32 q0, q0, q15\n\t" + "veor.32 q14, q14, q0\n\t" + "sub r12, r12, #16\n\t" + "vst1.8 {q14}, [%[out]]!\n\t" + "mov lr, #1\n\t" + "vrev32.8 q0, q2\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_192_done_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_ctr_encrypt_arm32_crypto_192_partial_done_%=\n\t" + "ldr r4, [sp, #8]\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "adds r8, r8, lr\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r7, r7, #0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r6, r6, #0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "adc r5, r5, #0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d5, r7, r8\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d4, r5, r6\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q11\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q12\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aese.8 q0, q13\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q15}, [r12]\n\t" + "ldr lr, [sp, #4]\n\t" + "aese.8 q0, q14\n\t" + "veor.32 q0, q0, q15\n\t" + "vst1.32 {q0}, [lr]\n\t" + "vmov q0, q2\n\t" + "mov r5, #16\n\t" + "sub r5, r5, %[sz]\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_192_start_byte_%=:\n\t" + "ldrb r7, [lr], #1\n\t" + "ldrb r8, [%[in]], #1\n\t" + "eor r7, r7, r8\n\t" + "subs %[sz], %[sz], #1\n\t" + "strb r7, [%[out]], #1\n\t" + "bgt L_aes_ctr_encrypt_arm32_crypto_192_start_byte_%=\n\t" + "vrev32.8 q0, q2\n\t" + "str r5, [r4]\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_192_partial_done_%=:\n\t" +#endif /* !NO_AES_192 */ + "b L_aes_ctr_encrypt_arm32_crypto_done_%=\n\t" + /* AES_CTR_256 */ + "\n" + "L_aes_ctr_encrypt_arm32_crypto_start_256_%=:\n\t" +#ifndef NO_AES_256 + "vldm.32 r12!, {q11-q13}\n\t" + "mov lr, #1\n\t" + "cmp r4, #1\n\t" + "blt L_aes_ctr_encrypt_arm32_crypto_256_done_%=\n\t" + "beq L_aes_ctr_encrypt_arm32_crypto_256_start_1_%=\n\t" + "adds r8, r8, #1\n\t" + "adcs r7, r7, #0\n\t" + "adcs r6, r6, #0\n\t" + "adc r5, r5, #0\n\t" + "vmov d3, r7, r8\n\t" + "vmov d2, r5, r6\n\t" + "vrev32.8 q1, q1\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_256_start_2_%=:\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "adds r8, r8, #1\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "adcs r7, r7, #0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "adcs r6, r6, #0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "adc r5, r5, #0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q0, q11\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q11\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q0, q12\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q12\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "vmov d5, r7, r8\n\t" + "aese.8 q0, q13\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q13\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q15}, [r12]!\n\t" + "vmov d4, r5, r6\n\t" + "aese.8 q0, q14\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q14\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "adds r8, r8, #1\n\t" + "aese.8 q0, q15\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q15\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q15}, [r12]\n\t" + "adcs r7, r7, #0\n\t" + "aese.8 q0, q14\n\t" + "aese.8 q1, q14\n\t" + "sub r4, r4, #2\n\t" + "veor.32 q0, q0, q15\n\t" + "veor.32 q1, q1, q15\n\t" + "adcs r6, r6, #0\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "sub r12, r12, #48\n\t" + "veor.32 q14, q14, q0\n\t" + "veor.32 q15, q15, q1\n\t" + "adc r5, r5, #0\n\t" + "vrev32.8 q0, q2\n\t" + "vmov d2, r5, r6\n\t" + "vmov d3, r7, r8\n\t" + "cmp r4, #1\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "vrev32.8 q1, q1\n\t" + "bgt L_aes_ctr_encrypt_arm32_crypto_256_start_2_%=\n\t" + "mov lr, #0\n\t" + "blt L_aes_ctr_encrypt_arm32_crypto_256_done_%=\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_256_start_1_%=:\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "adds r8, r8, lr\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r7, r7, #0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r6, r6, #0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "adc r5, r5, #0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d5, r7, r8\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d4, r5, r6\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q11\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q12\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aese.8 q0, q13\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q15}, [r12]!\n\t" + "aese.8 q0, q14\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aese.8 q0, q15\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q15}, [r12]\n\t" + "aese.8 q0, q14\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "veor.32 q0, q0, q15\n\t" + "veor.32 q14, q14, q0\n\t" + "sub r12, r12, #48\n\t" + "vst1.8 {q14}, [%[out]]!\n\t" + "mov lr, #1\n\t" + "vrev32.8 q0, q2\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_256_done_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_ctr_encrypt_arm32_crypto_256_partial_done_%=\n\t" + "ldr r4, [sp, #8]\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "adds r8, r8, lr\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r7, r7, #0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r6, r6, #0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "adc r5, r5, #0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d5, r7, r8\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d4, r5, r6\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q11\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q12\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aese.8 q0, q13\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q15}, [r12]!\n\t" + "aese.8 q0, q14\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aese.8 q0, q15\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q15}, [r12]\n\t" + "ldr lr, [sp, #4]\n\t" + "aese.8 q0, q14\n\t" + "veor.32 q0, q0, q15\n\t" + "vst1.32 {q0}, [lr]\n\t" + "vmov q0, q2\n\t" + "mov r5, #16\n\t" + "sub r5, r5, %[sz]\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_256_start_byte_%=:\n\t" + "ldrb r7, [lr], #1\n\t" + "ldrb r8, [%[in]], #1\n\t" + "eor r7, r7, r8\n\t" + "subs %[sz], %[sz], #1\n\t" + "strb r7, [%[out]], #1\n\t" + "bgt L_aes_ctr_encrypt_arm32_crypto_256_start_byte_%=\n\t" + "vrev32.8 q0, q2\n\t" + "str r5, [r4]\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_256_partial_done_%=:\n\t" +#endif /* !NO_AES_256 */ + "b L_aes_ctr_encrypt_arm32_crypto_done_%=\n\t" + /* AES_CTR_128 */ + "\n" + "L_aes_ctr_encrypt_arm32_crypto_start_128_%=:\n\t" +#ifndef NO_AES_128 + "vldm.32 r12!, {q11-q13}\n\t" + "mov lr, #1\n\t" + "cmp r4, #1\n\t" + "blt L_aes_ctr_encrypt_arm32_crypto_128_done_%=\n\t" + "beq L_aes_ctr_encrypt_arm32_crypto_128_start_1_%=\n\t" + "adds r8, r8, #1\n\t" + "adcs r7, r7, #0\n\t" + "adcs r6, r6, #0\n\t" + "adc r5, r5, #0\n\t" + "vmov d3, r7, r8\n\t" + "vmov d2, r5, r6\n\t" + "vrev32.8 q1, q1\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_128_start_2_%=:\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "adds r8, r8, #1\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "adcs r7, r7, #0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "adcs r6, r6, #0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "adc r5, r5, #0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "vmov d5, r7, r8\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "vmov d4, r5, r6\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "adds r8, r8, #1\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "adcs r7, r7, #0\n\t" + "aese.8 q0, q11\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q1, q11\n\t" + "aesmc.8 q1, q1\n\t" + "adcs r6, r6, #0\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "adc r5, r5, #0\n\t" + "aese.8 q0, q12\n\t" + "aese.8 q1, q12\n\t" + "sub r4, r4, #2\n\t" + "veor.32 q14, q14, q13\n\t" + "veor.32 q15, q15, q13\n\t" + "veor.32 q14, q14, q0\n\t" + "veor.32 q15, q15, q1\n\t" + "vrev32.8 q0, q2\n\t" + "vmov d2, r5, r6\n\t" + "vmov d3, r7, r8\n\t" + "cmp r4, #1\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "vrev32.8 q1, q1\n\t" + "bgt L_aes_ctr_encrypt_arm32_crypto_128_start_2_%=\n\t" + "mov lr, #0\n\t" + "blt L_aes_ctr_encrypt_arm32_crypto_128_done_%=\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_128_start_1_%=:\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "adds r8, r8, lr\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r7, r7, #0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r6, r6, #0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "adc r5, r5, #0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d5, r7, r8\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d4, r5, r6\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q11\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q0, q12\n\t" + "veor.32 q0, q0, q13\n\t" + "veor.32 q14, q14, q0\n\t" + "vst1.8 {q14}, [%[out]]!\n\t" + "mov lr, #1\n\t" + "vrev32.8 q0, q2\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_128_done_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_ctr_encrypt_arm32_crypto_128_partial_done_%=\n\t" + "ldr r4, [sp, #8]\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "adds r8, r8, lr\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r7, r7, #0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "adcs r6, r6, #0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "adc r5, r5, #0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d5, r7, r8\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "vmov d4, r5, r6\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q11\n\t" + "aesmc.8 q0, q0\n\t" + "ldr lr, [sp, #4]\n\t" + "aese.8 q0, q12\n\t" + "veor.32 q0, q0, q13\n\t" + "vst1.32 {q0}, [lr]\n\t" + "vmov q0, q2\n\t" + "mov r5, #16\n\t" + "sub r5, r5, %[sz]\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_128_start_byte_%=:\n\t" + "ldrb r7, [lr], #1\n\t" + "ldrb r8, [%[in]], #1\n\t" + "eor r7, r7, r8\n\t" + "subs %[sz], %[sz], #1\n\t" + "strb r7, [%[out]], #1\n\t" + "bgt L_aes_ctr_encrypt_arm32_crypto_128_start_byte_%=\n\t" + "vrev32.8 q0, q2\n\t" + "str r5, [r4]\n\t" + "\n" + "L_aes_ctr_encrypt_arm32_crypto_128_partial_done_%=:\n\t" +#endif /* !NO_AES_128 */ + "\n" + "L_aes_ctr_encrypt_arm32_crypto_done_%=:\n\t" + "vst1.32 {q0}, [%[reg]]\n\t" + "pop {%[key], %[tmp]}\n\t" + "pop {%[left], %[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [reg] "+r" (reg), + [key] "+r" (key), [tmp] "+r" (tmp), [left] "+r" (left), + [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "uxtb lr, r9, ror #16\n\t" -#endif + : + : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [reg] "r" (reg), + [key] "r" (key), [tmp] "r" (tmp), [left] "r" (left), [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r7", "r8", "q0", "q1", "q2", "q3", "q4", "q5", "q6", + "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#endif /* WOLFSSL_AES_COUNTER */ +#ifdef HAVE_AESGCM +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_GCM_set_key_AARCH32(const byte* nonce_p, + const byte* key_p, byte* gcm_h_p, int nr_p) #else - "ubfx lr, r9, #16, #8\n\t" -#endif - "lsr r2, r8, #24\n\t" - "ldrb r4, [r0, r4, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r5, r8, #24\n\t" - "lsr r5, r5, #24\n\t" +WC_OMIT_FRAME_POINTER void AES_GCM_set_key_AARCH32(const byte* nonce, + const byte* key, byte* gcm_h, int nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* nonce __asm__ ("r0") = (const byte*)nonce_p; + register const byte* key __asm__ ("r1") = (const byte*)key_p; + register byte* gcm_h __asm__ ("r2") = (byte*)gcm_h_p; + register int nr __asm__ ("r3") = (int)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r12") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "uxtb r5, r8\n\t" -#endif + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_GCMSIV_ctr_base_te]}\n\t" + "vld1.8 {q0}, [%[nonce]]\n\t" + "vld1.8 {q1-q2}, [%[key]]!\n\t" + "vld1.8 {q3-q4}, [%[key]]!\n\t" + "aese.8 q0, q1\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q2\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.8 {q1-q2}, [%[key]]!\n\t" + "vld1.8 {q3-q4}, [%[key]]!\n\t" + "aese.8 q0, q1\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q2\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "subs %[nr], %[nr], #10\n\t" + "vld1.8 {q1-q2}, [%[key]]!\n\t" + "aese.8 q0, q1\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q2\n\t" + "beq L_aes_gcm_set_key_arm32_crypto_round_done_%=\n\t" + "vld1.8 {q1-q2}, [%[key]]!\n\t" + "subs %[nr], %[nr], #2\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q1\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q2\n\t" + "beq L_aes_gcm_set_key_arm32_crypto_round_done_%=\n\t" + "vld1.8 {q1-q2}, [%[key]]!\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q1\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q2\n\t" + "\n" + "L_aes_gcm_set_key_arm32_crypto_round_done_%=:\n\t" + "vld1.8 {q1}, [%[key]]\n\t" + "veor q0, q0, q1\n\t" + "vmov.i8 q1, #0x55\n\t" + "vshl.u8 q2, q0, #1\n\t" + "vshr.u8 q0, q0, #1\n\t" + "vbif.8 q0, q2, q1\n\t" + "vmov.i8 q1, #51\n\t" + "vshl.u8 q2, q0, #2\n\t" + "vshr.u8 q0, q0, #2\n\t" + "vbit.8 q2, q0, q1\n\t" + "vshl.u8 q0, q2, #4\n\t" + "vsri.u8 q0, q2, #4\n\t" + "vst1.32 {q0}, [%[gcm_h]]\n\t" + "pop {%[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [nonce] "+r" (nonce), [key] "+r" (key), [gcm_h] "+r" (gcm_h), + [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "ubfx r5, r8, #0, #8\n\t" -#endif - "eor r4, r4, r7, lsl #8\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r11, #16\n\t" - "lsr r7, r7, #24\n\t" + : + : [nonce] "r" (nonce), [key] "r" (key), [gcm_h] "r" (gcm_h), + [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "q0", "q1", "q2", "q3", "q4" + ); +} + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_GCM_encrypt_AARCH32(const byte* in_p, + byte* out_p, word32 sz_p, const byte* nonce_p, word32 nonceSz_p, + byte* tag_p, word32 tagSz_p, const byte* aad_p, word32 aadSz_p, byte* key_p, + byte* gcm_h_p, byte* tmp_p, byte* reg_p, int nr_p) #else - "uxtb r7, r11, ror #8\n\t" -#endif +WC_OMIT_FRAME_POINTER void AES_GCM_encrypt_AARCH32(const byte* in, byte* out, + word32 sz, const byte* nonce, word32 nonceSz, byte* tag, word32 tagSz, + const byte* aad, word32 aadSz, byte* key, byte* gcm_h, byte* tmp, byte* reg, + int nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* in __asm__ ("r0") = (const byte*)in_p; + register byte* out __asm__ ("r1") = (byte*)out_p; + register word32 sz __asm__ ("r2") = (word32)sz_p; + register const byte* nonce __asm__ ("r3") = (const byte*)nonce_p; + register word32 nonceSz __asm__ ("r12") = (word32)nonceSz_p; + register byte* tag __asm__ ("lr") = (byte*)tag_p; + register word32 tagSz __asm__ ("r4") = (word32)tagSz_p; + register const byte* aad __asm__ ("r5") = (const byte*)aad_p; + register word32 aadSz __asm__ ("r6") = (word32)aadSz_p; + register byte* key __asm__ ("r7") = (byte*)key_p; + register byte* gcm_h __asm__ ("r8") = (byte*)gcm_h_p; + register byte* tmp __asm__ ("r9") = (byte*)tmp_p; + register byte* reg __asm__ ("r10") = (byte*)reg_p; + register int nr __asm__ ("r11") = (int)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "ubfx r7, r11, #8, #8\n\t" -#endif - "eor r4, r4, lr, lsl #16\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r10, #8\n\t" - "lsr lr, lr, #24\n\t" + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[tagSz], %[aad], %[aadSz], %[key], %[gcm_h], %[tmp], %[reg], %[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" + "push {%[nonceSz], %[tag]}\n\t" + /* key */ + "ldr r7, [sp, #20]\n\t" + /* tmp */ + "ldr r9, [sp, #28]\n\t" + /* nonceSz */ + "ldr r12, [sp]\n\t" + "cmp r12, #12\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_nonce_setup_done_%=\n\t" + "vmov.i8 q13, #0x87\n\t" + /* gcm_h */ + "ldr r8, [sp, #24]\n\t" + "veor.8 q6, q6, q6\n\t" + "vshr.u64 q13, q13, #56\n\t" + "vld1.32 {q8}, [r8]\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_nonce_setup_done_%=:\n\t" + /* Load Nonce */ + "cmp r12, #12\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_ghash_nonce_%=\n\t" + "ldr r5, [%[nonce]]\n\t" + "ldr r8, [%[nonce], #4]\n\t" + "ldr r12, [%[nonce], #8]\n\t" + "vmov.i32 q6, #0x1000000\n\t" + "vmov.32 s24, r5\n\t" + "vmov.32 s25, r8\n\t" + "vmov.32 s26, r12\n\t" + "mov r5, #1\n\t" + "b L_aes_gcm_encrypt_arm32_crypto_done_nonce_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_ghash_nonce_%=:\n\t" + "lsr r10, r12, #4\n\t" + "cmp r10, #0\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_nonce_done_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_nonce_start_1_%=:\n\t" + "vld1.32 {q14}, [%[nonce]]!\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q6, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q6, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q6\n\t" + /* Reduce */ + "vmull.p64 q6, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d13\n\t" + "veor.8 d1, d1, d12\n\t" + "vmull.p64 q6, d2, d26\n\t" + "veor.8 q6, q6, q0\n\t" + /* Done GHASH */ + "subs r10, r10, #1\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_nonce_start_1_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_nonce_done_%=:\n\t" + "ands r11, r12, #15\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_nonce_partial_done_%=\n\t" + "veor.8 q0, q0, q0\n\t" + "mov r12, r11\n\t" + "vst1.32 {q0}, [r9]\n\t" + "cmp r12, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_nonce_start_sw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_nonce_start_dw_%=:\n\t" + "ldr r8, [%[nonce]], #4\n\t" + "sub r12, r12, #4\n\t" + "str r8, [r9], #4\n\t" + "cmp r12, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_nonce_start_dw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_nonce_start_sw_%=:\n\t" + "cmp r12, #2\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte_%=\n\t" + "ldrh r8, [%[nonce]], #2\n\t" + "sub r12, r12, #2\n\t" + "strh r8, [r9], #2\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte_%=:\n\t" + "cmp r12, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_nonce_end_bytes_%=\n\t" + "ldrb r8, [%[nonce]], #1\n\t" + "subs r12, r12, #1\n\t" + "strb r8, [r9], #1\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_nonce_start_byte_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_nonce_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "vld1.32 {q14}, [r9]\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q6, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q6, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q6\n\t" + /* Reduce */ + "vmull.p64 q6, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d13\n\t" + "veor.8 d1, d1, d12\n\t" + "vmull.p64 q6, d2, d26\n\t" + "veor.8 q6, q6, q0\n\t" + /* Done GHASH */ + "\n" + "L_aes_gcm_encrypt_arm32_crypto_nonce_partial_done_%=:\n\t" + "veor.8 q0, q0, q0\n\t" + /* nonceSz */ + "ldr r12, [sp]\n\t" + "lsr r10, r12, #29\n\t" + "lsl r11, r12, #3\n\t" + "rbit r10, r10\n\t" + "rbit r11, r11\n\t" + "vmov.32 s2, r10\n\t" + "vmov.32 s3, r11\n\t" + "veor.8 q6, q6, q0\n\t" + "vmull.p64 q2, d13, d16\n\t" + "vmull.p64 q12, d12, d17\n\t" + "vmull.p64 q0, d12, d16\n\t" + "vmull.p64 q1, d13, d17\n\t" + "veor.8 q2, q2, q12\n\t" + "vmull.p64 q6, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d13\n\t" + "veor.8 d1, d1, d12\n\t" + "vmull.p64 q6, d2, d26\n\t" + "veor.8 q6, q6, q0\n\t" + "vmov.32 r5, s24\n\t" + "vmov.32 r8, s25\n\t" + "vmov.32 r12, s26\n\t" + "rbit r5, r5\n\t" + "rbit r8, r8\n\t" + "rbit r12, r12\n\t" + "rev r5, r5\n\t" + "rev r8, r8\n\t" + "rev r12, r12\n\t" + "vmov.32 s24, r5\n\t" + "vmov.32 s25, r8\n\t" + "vmov.32 s26, r12\n\t" + "vmov.32 r5, s27\n\t" + "rbit r5, r5\n\t" + "rev r5, r5\n\t" + "vmov.32 s27, r5\n\t" + "rev r5, r5\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_done_nonce_%=:\n\t" + "vldm.32 r7!, {q0-q3}\n\t" + "vldm.32 r7!, {q7-q13}\n\t" + /* nr */ + "ldr r12, [sp, #36]\n\t" + "lsr r10, %[sz], #4\n\t" + "cmp r12, #12\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_start_128_%=\n\t" + "bgt L_aes_gcm_encrypt_arm32_crypto_start_256_%=\n\t" + /* AES_GCM_192 */ +#ifndef NO_AES_192 + "cmp r10, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_192_done_%=\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_192_start_1_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_start_2_%=:\n\t" + "add r8, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "add r5, r5, #2\n\t" + "vmov.8 q5, q6\n\t" + "rev r8, r8\n\t" + "rev r12, r5\n\t" + "vmov s19, r8\n\t" + "vmov s23, r12\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q0\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q1\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q2\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q3\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q7\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q8\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q9\n\t" + "aesmc.8 q5, q5\n\t" + "subs r10, r10, #2\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q10\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q11\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q12\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q13\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "aese.8 q5, q14\n\t" + "veor.8 q5, q5, q15\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "sub r7, r7, #16\n\t" + "veor.8 q14, q14, q4\n\t" + "veor.8 q15, q15, q5\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "cmp r10, #1\n\t" + "bgt L_aes_gcm_encrypt_arm32_crypto_192_start_2_%=\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_192_done_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_start_1_%=:\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "sub r7, r7, #16\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [%[out]]!\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_done_%=:\n\t" + "ands r11, %[sz], #15\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_192_partial_done_%=\n\t" + "veor.8 q14, q14, q14\n\t" + "mov r4, r11\n\t" + "vst1.32 {q14}, [r9]\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_192_start_sw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_start_dw_%=:\n\t" + "ldr lr, [%[in]], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [r9], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_192_start_dw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_192_start_byte_%=\n\t" + "ldrh lr, [%[in]], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [r9], #2\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_192_end_bytes_%=\n\t" + "ldrb lr, [%[in]], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [r9], #1\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_192_start_byte_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "vld1.8 {q14}, [r9]\n\t" + "sub r7, r7, #16\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [r9]\n\t" + "mov r4, r11\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_192_out_start_sw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_out_start_dw_%=:\n\t" + "ldr lr, [r9], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [%[out]], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_192_out_start_dw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_out_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte_%=\n\t" + "ldrh lr, [r9], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [%[out]], #2\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_192_out_end_bytes_%=\n\t" + "ldrb lr, [r9], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [%[out]], #1\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_192_out_start_byte_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_out_end_bytes_%=:\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_192_partial_done_%=:\n\t" + /* Finish */ + "add r8, %[sz], #15\n\t" + "sub r8, r5, r8, lsr #4\n\t" + "rev r8, r8\n\t" + "vmov.32 s27, r8\n\t" + "aese.8 q6, q0\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q1\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q2\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q3\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q7\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q8\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q9\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q10\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q11\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q12\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q6, q13\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q6, q14\n\t" + "veor.8 q6, q6, q15\n\t" + "sub r7, r7, #16\n\t" +#endif /* !NO_AES_192 */ + "b L_aes_gcm_encrypt_arm32_crypto_done_enc_%=\n\t" + /* AES_GCM_256 */ + "\n" + "L_aes_gcm_encrypt_arm32_crypto_start_256_%=:\n\t" +#ifndef NO_AES_256 + "cmp r10, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_256_done_%=\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_256_start_1_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_start_2_%=:\n\t" + "add r8, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "add r5, r5, #2\n\t" + "vmov.8 q5, q6\n\t" + "rev r8, r8\n\t" + "rev r12, r5\n\t" + "vmov s19, r8\n\t" + "vmov s23, r12\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q0\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q1\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q2\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q3\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q7\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q8\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q9\n\t" + "aesmc.8 q5, q5\n\t" + "subs r10, r10, #2\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q10\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q11\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q12\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q13\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q15}, [r7]!\n\t" + "aese.8 q4, q14\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q14\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q15\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q15\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "aese.8 q5, q14\n\t" + "veor.8 q5, q5, q15\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "sub r7, r7, #48\n\t" + "veor.8 q14, q14, q4\n\t" + "veor.8 q15, q15, q5\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "cmp r10, #1\n\t" + "bgt L_aes_gcm_encrypt_arm32_crypto_256_start_2_%=\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_256_done_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_start_1_%=:\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]!\n\t" + "aese.8 q4, q14\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q15\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "sub r7, r7, #48\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [%[out]]!\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_done_%=:\n\t" + "ands r11, %[sz], #15\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_256_partial_done_%=\n\t" + "veor.8 q14, q14, q14\n\t" + "mov r4, r11\n\t" + "vst1.32 {q14}, [r9]\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_256_start_sw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_start_dw_%=:\n\t" + "ldr lr, [%[in]], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [r9], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_256_start_dw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_256_start_byte_%=\n\t" + "ldrh lr, [%[in]], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [r9], #2\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_256_end_bytes_%=\n\t" + "ldrb lr, [%[in]], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [r9], #1\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_256_start_byte_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]!\n\t" + "aese.8 q4, q14\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q15\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "vld1.8 {q14}, [r9]\n\t" + "sub r7, r7, #48\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [r9]\n\t" + "mov r4, r11\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_256_out_start_sw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_out_start_dw_%=:\n\t" + "ldr lr, [r9], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [%[out]], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_256_out_start_dw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_out_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte_%=\n\t" + "ldrh lr, [r9], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [%[out]], #2\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_256_out_end_bytes_%=\n\t" + "ldrb lr, [r9], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [%[out]], #1\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_256_out_start_byte_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_out_end_bytes_%=:\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_256_partial_done_%=:\n\t" + /* Finish */ + "add r8, %[sz], #15\n\t" + "sub r8, r5, r8, lsr #4\n\t" + "rev r8, r8\n\t" + "vmov.32 s27, r8\n\t" + "aese.8 q6, q0\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q1\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q2\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q3\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q7\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q8\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q9\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q10\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q11\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q12\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q6, q13\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q15}, [r7]!\n\t" + "aese.8 q6, q14\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q6, q15\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q6, q14\n\t" + "veor.8 q6, q6, q15\n\t" + "sub r7, r7, #48\n\t" +#endif /* !NO_AES_256 */ + "b L_aes_gcm_encrypt_arm32_crypto_done_enc_%=\n\t" + /* AES_GCM_128 */ + "\n" + "L_aes_gcm_encrypt_arm32_crypto_start_128_%=:\n\t" +#ifndef NO_AES_128 + "cmp r10, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_128_done_%=\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_128_start_1_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_start_2_%=:\n\t" + "add r8, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "add r5, r5, #2\n\t" + "vmov.8 q5, q6\n\t" + "rev r8, r8\n\t" + "rev r12, r5\n\t" + "vmov s19, r8\n\t" + "vmov s23, r12\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q0\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q1\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q2\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q3\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q7\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q8\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q9\n\t" + "aesmc.8 q5, q5\n\t" + "subs r10, r10, #2\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q10\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q11\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "aese.8 q4, q12\n\t" + "veor.8 q4, q4, q13\n\t" + "aese.8 q5, q12\n\t" + "veor.8 q5, q5, q13\n\t" + "veor.8 q14, q14, q4\n\t" + "veor.8 q15, q15, q5\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "cmp r10, #1\n\t" + "bgt L_aes_gcm_encrypt_arm32_crypto_128_start_2_%=\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_128_done_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_start_1_%=:\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q4, q12\n\t" + "veor.8 q4, q4, q13\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [%[out]]!\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_done_%=:\n\t" + "ands r11, %[sz], #15\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_128_partial_done_%=\n\t" + "veor.8 q14, q14, q14\n\t" + "mov r4, r11\n\t" + "vst1.32 {q14}, [r9]\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_128_start_sw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_start_dw_%=:\n\t" + "ldr lr, [%[in]], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [r9], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_128_start_dw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_128_start_byte_%=\n\t" + "ldrh lr, [%[in]], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [r9], #2\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_128_end_bytes_%=\n\t" + "ldrb lr, [%[in]], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [r9], #1\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_128_start_byte_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.8 {q14}, [r9]\n\t" + "aese.8 q4, q12\n\t" + "veor.8 q4, q4, q13\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [r9]\n\t" + "mov r4, r11\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_128_out_start_sw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_out_start_dw_%=:\n\t" + "ldr lr, [r9], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [%[out]], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_128_out_start_dw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_out_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte_%=\n\t" + "ldrh lr, [r9], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [%[out]], #2\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_128_out_end_bytes_%=\n\t" + "ldrb lr, [r9], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [%[out]], #1\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_128_out_start_byte_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_out_end_bytes_%=:\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_128_partial_done_%=:\n\t" + /* Finish */ + "add r8, %[sz], #15\n\t" + "sub r8, r5, r8, lsr #4\n\t" + "rev r8, r8\n\t" + "vmov.32 s27, r8\n\t" + "aese.8 q6, q0\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q1\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q2\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q3\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q7\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q8\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q9\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q10\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q11\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q12\n\t" + "veor.8 q6, q6, q13\n\t" +#endif /* !NO_AES_128 */ + "\n" + "L_aes_gcm_encrypt_arm32_crypto_done_enc_%=:\n\t" + /* aadSz */ + "ldr r6, [sp, #16]\n\t" + /* gcm_h */ + "ldr r8, [sp, #24]\n\t" + "sub %[out], %[out], %[sz]\n\t" + "vmov.i8 q13, #0x87\n\t" + "veor.8 q7, q7, q7\n\t" + "vshr.u64 q13, q13, #56\n\t" + "vld1.32 {q8}, [r8]\n\t" + "orr r10, r6, %[sz]\n\t" + "cmp r10, #32\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_h_done_%=\n\t" + /* Square H => H^2 */ + "vmull.p64 q1, d17, d17\n\t" + "vmull.p64 q0, d16, d16\n\t" + "vmull.p64 q9, d3, d27\n\t" + "veor.8 d2, d2, d19\n\t" + "veor.8 d1, d1, d18\n\t" + "vmull.p64 q9, d2, d26\n\t" + "veor.8 q9, q9, q0\n\t" + "cmp r10, #0x40\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_h_done_%=\n\t" + /* Multiply H and H^2 => H^3 */ + "vmull.p64 q2, d17, d18\n\t" + "vmull.p64 q3, d16, d19\n\t" + "vmull.p64 q0, d16, d18\n\t" + "vmull.p64 q1, d17, d19\n\t" + "veor.8 q2, q2, q3\n\t" + /* Reduce */ + "vmull.p64 q10, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d21\n\t" + "veor.8 d1, d1, d20\n\t" + "vmull.p64 q10, d2, d26\n\t" + "veor.8 q10, q10, q0\n\t" + /* Square H^2 => H^4 */ + "vmull.p64 q1, d19, d19\n\t" + "vmull.p64 q0, d18, d18\n\t" + "vmull.p64 q11, d3, d27\n\t" + "veor.8 d2, d2, d23\n\t" + "veor.8 d1, d1, d22\n\t" + "vmull.p64 q11, d2, d26\n\t" + "veor.8 q11, q11, q0\n\t" + /* Done */ + "\n" + "L_aes_gcm_encrypt_arm32_crypto_h_done_%=:\n\t" + /* aad */ + "ldr r5, [sp, #12]\n\t" + "lsr r10, r6, #4\n\t" + "cmp r10, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_aad_done_%=\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_aad_start_1_%=\n\t" + "cmp r10, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_aad_start_2_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_aad_start_4_%=:\n\t" + "vldm r5!, {q0-q2}\n\t" + "vmov.i8 q12, #0x55\n\t" + "vmov.i8 q5, #51\n\t" + "vshl.u8 q14, q0, #1\n\t" + "vshl.u8 q15, q1, #1\n\t" + "vshl.u8 q4, q2, #1\n\t" + "ldr lr, [r5], #4\n\t" + "vshr.u8 q0, q0, #1\n\t" + "vshr.u8 q1, q1, #1\n\t" + "vshr.u8 q2, q2, #1\n\t" + "ldr r4, [r5], #4\n\t" + "vbit.8 q14, q0, q12\n\t" + "vbit.8 q15, q1, q12\n\t" + "vbit.8 q4, q2, q12\n\t" + "ldr r8, [r5], #4\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshl.u8 q1, q15, #2\n\t" + "vshl.u8 q2, q4, #2\n\t" + "ldr r12, [r5], #4\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vshr.u8 q15, q15, #2\n\t" + "vshr.u8 q4, q4, #2\n\t" + "rbit lr, lr\n\t" + "vbit.8 q0, q14, q5\n\t" + "rbit r4, r4\n\t" + "vbit.8 q1, q15, q5\n\t" + "rbit r8, r8\n\t" + "vbit.8 q2, q4, q5\n\t" + "rbit r12, r12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "rev lr, lr\n\t" + "vshl.u8 q15, q1, #4\n\t" + "rev r4, r4\n\t" + "vshl.u8 q4, q2, #4\n\t" + "rev r8, r8\n\t" + "vsri.u8 q14, q0, #4\n\t" + "rev r12, r12\n\t" + "vsri.u8 q15, q1, #4\n\t" + "vsri.u8 q4, q2, #4\n\t" + "vmov d10, lr, r4\n\t" + "vmov d11, r8, r12\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d11, d16\n\t" + "vmull.p64 q7, d10, d17\n\t" + "vmull.p64 q0, d10, d16\n\t" + "vmull.p64 q1, d11, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^2 */ + "vmull.p64 q3, d8, d18\n\t" + "vmull.p64 q7, d9, d19\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d9, d18\n\t" + "vmull.p64 q7, d8, d19\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^3 */ + "vmull.p64 q3, d30, d20\n\t" + "vmull.p64 q7, d31, d21\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d31, d20\n\t" + "vmull.p64 q7, d30, d21\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^4 */ + "vmull.p64 q3, d24, d22\n\t" + "vmull.p64 q7, d25, d23\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d25, d22\n\t" + "vmull.p64 q7, d24, d23\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "sub r10, r10, #4\n\t" + "cmp r10, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_aad_start_4_%=\n\t" + "cmp r10, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_aad_done_%=\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_aad_start_1_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_aad_start_2_%=:\n\t" + "vld1.32 {q14-q15}, [r5]!\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshl.u8 q1, q15, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vshr.u8 q15, q15, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vbif.8 q15, q1, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshl.u8 q1, q15, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vshr.u8 q15, q15, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vbit.8 q1, q15, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vshl.u8 q15, q1, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "vsri.u8 q15, q1, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d31, d16\n\t" + "vmull.p64 q7, d30, d17\n\t" + "vmull.p64 q0, d30, d16\n\t" + "vmull.p64 q1, d31, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^2 */ + "vmull.p64 q3, d24, d18\n\t" + "vmull.p64 q7, d25, d19\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d25, d18\n\t" + "vmull.p64 q7, d24, d19\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "sub r10, r10, #2\n\t" + "cmp r10, #0\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_aad_done_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_aad_start_1_%=:\n\t" + "vld1.32 {q14}, [r5]!\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q7, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "\n" + "L_aes_gcm_encrypt_arm32_crypto_aad_done_%=:\n\t" + "ands r11, r6, #15\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_aad_partial_done_%=\n\t" + "veor.8 q0, q0, q0\n\t" + "mov r12, r11\n\t" + "vst1.32 {q0}, [r9]\n\t" + "cmp r12, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_aad_start_sw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_aad_start_dw_%=:\n\t" + "ldr r8, [r5], #4\n\t" + "sub r12, r12, #4\n\t" + "str r8, [r9], #4\n\t" + "cmp r12, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_aad_start_dw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_aad_start_sw_%=:\n\t" + "cmp r12, #2\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_aad_start_byte_%=\n\t" + "ldrh r8, [r5], #2\n\t" + "sub r12, r12, #2\n\t" + "strh r8, [r9], #2\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_aad_start_byte_%=:\n\t" + "cmp r12, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_aad_end_bytes_%=\n\t" + "ldrb r8, [r5], #1\n\t" + "subs r12, r12, #1\n\t" + "strb r8, [r9], #1\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_aad_start_byte_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_aad_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "vld1.32 {q14}, [r9]\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q7, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "\n" + "L_aes_gcm_encrypt_arm32_crypto_aad_partial_done_%=:\n\t" + /* out */ + "lsr r10, %[sz], #4\n\t" + "cmp r10, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_out_done_%=\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_out_start_1_%=\n\t" + "cmp r10, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_out_start_2_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_out_start_4_%=:\n\t" + "vldm %[out]!, {q0-q2}\n\t" + "vmov.i8 q12, #0x55\n\t" + "vmov.i8 q5, #51\n\t" + "vshl.u8 q14, q0, #1\n\t" + "vshl.u8 q15, q1, #1\n\t" + "vshl.u8 q4, q2, #1\n\t" + "ldr lr, [%[out]], #4\n\t" + "vshr.u8 q0, q0, #1\n\t" + "vshr.u8 q1, q1, #1\n\t" + "vshr.u8 q2, q2, #1\n\t" + "ldr r4, [%[out]], #4\n\t" + "vbit.8 q14, q0, q12\n\t" + "vbit.8 q15, q1, q12\n\t" + "vbit.8 q4, q2, q12\n\t" + "ldr r8, [%[out]], #4\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshl.u8 q1, q15, #2\n\t" + "vshl.u8 q2, q4, #2\n\t" + "ldr r12, [%[out]], #4\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vshr.u8 q15, q15, #2\n\t" + "vshr.u8 q4, q4, #2\n\t" + "rbit lr, lr\n\t" + "vbit.8 q0, q14, q5\n\t" + "rbit r4, r4\n\t" + "vbit.8 q1, q15, q5\n\t" + "rbit r8, r8\n\t" + "vbit.8 q2, q4, q5\n\t" + "rbit r12, r12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "rev lr, lr\n\t" + "vshl.u8 q15, q1, #4\n\t" + "rev r4, r4\n\t" + "vshl.u8 q4, q2, #4\n\t" + "rev r8, r8\n\t" + "vsri.u8 q14, q0, #4\n\t" + "rev r12, r12\n\t" + "vsri.u8 q15, q1, #4\n\t" + "vsri.u8 q4, q2, #4\n\t" + "vmov d10, lr, r4\n\t" + "vmov d11, r8, r12\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d11, d16\n\t" + "vmull.p64 q7, d10, d17\n\t" + "vmull.p64 q0, d10, d16\n\t" + "vmull.p64 q1, d11, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^2 */ + "vmull.p64 q3, d8, d18\n\t" + "vmull.p64 q7, d9, d19\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d9, d18\n\t" + "vmull.p64 q7, d8, d19\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^3 */ + "vmull.p64 q3, d30, d20\n\t" + "vmull.p64 q7, d31, d21\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d31, d20\n\t" + "vmull.p64 q7, d30, d21\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^4 */ + "vmull.p64 q3, d24, d22\n\t" + "vmull.p64 q7, d25, d23\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d25, d22\n\t" + "vmull.p64 q7, d24, d23\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "sub r10, r10, #4\n\t" + "cmp r10, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_out_start_4_%=\n\t" + "cmp r10, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_out_done_%=\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_out_start_1_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_out_start_2_%=:\n\t" + "vld1.32 {q14-q15}, [%[out]]!\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshl.u8 q1, q15, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vshr.u8 q15, q15, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vbif.8 q15, q1, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshl.u8 q1, q15, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vshr.u8 q15, q15, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vbit.8 q1, q15, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vshl.u8 q15, q1, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "vsri.u8 q15, q1, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d31, d16\n\t" + "vmull.p64 q7, d30, d17\n\t" + "vmull.p64 q0, d30, d16\n\t" + "vmull.p64 q1, d31, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^2 */ + "vmull.p64 q3, d24, d18\n\t" + "vmull.p64 q7, d25, d19\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d25, d18\n\t" + "vmull.p64 q7, d24, d19\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "sub r10, r10, #2\n\t" + "cmp r10, #0\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_out_done_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_out_start_1_%=:\n\t" + "vld1.32 {q14}, [%[out]]!\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q7, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "\n" + "L_aes_gcm_encrypt_arm32_crypto_out_done_%=:\n\t" + "ands r11, %[sz], #15\n\t" + "beq L_aes_gcm_encrypt_arm32_crypto_out_partial_done_%=\n\t" + "veor.8 q0, q0, q0\n\t" + "mov r12, r11\n\t" + "vst1.32 {q0}, [r9]\n\t" + "cmp r12, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_out_start_sw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_out_start_dw_%=:\n\t" + "ldr r8, [%[out]], #4\n\t" + "sub r12, r12, #4\n\t" + "str r8, [r9], #4\n\t" + "cmp r12, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_out_start_dw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_out_start_sw_%=:\n\t" + "cmp r12, #2\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_out_start_byte_%=\n\t" + "ldrh r8, [%[out]], #2\n\t" + "sub r12, r12, #2\n\t" + "strh r8, [r9], #2\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_out_start_byte_%=:\n\t" + "cmp r12, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_out_end_bytes_%=\n\t" + "ldrb r8, [%[out]], #1\n\t" + "subs r12, r12, #1\n\t" + "strb r8, [r9], #1\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_out_start_byte_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_out_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "vld1.32 {q14}, [r9]\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q7, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "\n" + "L_aes_gcm_encrypt_arm32_crypto_out_partial_done_%=:\n\t" + "lsr lr, r6, #29\n\t" + "lsl r6, r6, #3\n\t" + "rbit lr, lr\n\t" + "rbit r6, r6\n\t" + "vmov s0, lr\n\t" + "vmov s1, r6\n\t" + "lsr lr, %[sz], #29\n\t" + "lsl %[sz], %[sz], #3\n\t" + "rbit lr, lr\n\t" + "rbit %[sz], %[sz]\n\t" + "vmov s2, lr\n\t" + "vmov s3, %[sz]\n\t" + "veor.8 q7, q7, q0\n\t" + "vmull.p64 q2, d15, d16\n\t" + "vmull.p64 q3, d14, d17\n\t" + "vmull.p64 q0, d14, d16\n\t" + "vmull.p64 q1, d15, d17\n\t" + "veor.8 q2, q2, q3\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + "vmov.i8 q0, #0x55\n\t" + "vshl.u8 q1, q7, #1\n\t" + "vshr.u8 q7, q7, #1\n\t" + "vbif.8 q7, q1, q0\n\t" + "vmov.i8 q0, #51\n\t" + "vshl.u8 q1, q7, #2\n\t" + "vshr.u8 q7, q7, #2\n\t" + "vbit.8 q1, q7, q0\n\t" + "vshl.u8 q7, q1, #4\n\t" + "vsri.u8 q7, q1, #4\n\t" + /* DONE */ + "veor.8 q7, q7, q6\n\t" + /* tag */ + "ldr lr, [sp, #4]\n\t" + /* tagSz */ + "ldr r4, [sp, #8]\n\t" + "cmp r4, #16\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_tag_tag_partial_%=\n\t" + "vst1.8 {q7}, [lr]\n\t" + "b L_aes_gcm_encrypt_arm32_crypto_done_gcm_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_tag_tag_partial_%=:\n\t" + "vst1.8 {q7}, [r9]\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_sw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_dw_%=:\n\t" + "ldr r8, [r9], #4\n\t" + "sub r4, r4, #4\n\t" + "str r8, [lr], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_dw_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte_%=\n\t" + "ldrh r8, [r9], #2\n\t" + "sub r4, r4, #2\n\t" + "strh r8, [lr], #2\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_encrypt_arm32_crypto_tag_tag_end_bytes_%=\n\t" + "ldrb r8, [r9], #1\n\t" + "subs r4, r4, #1\n\t" + "strb r8, [lr], #1\n\t" + "bne L_aes_gcm_encrypt_arm32_crypto_tag_tag_start_byte_%=\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_tag_tag_end_bytes_%=:\n\t" + "\n" + "L_aes_gcm_encrypt_arm32_crypto_done_gcm_%=:\n\t" + "pop {%[nonceSz], %[tag]}\n\t" + "pop {%[tagSz], %[aad], %[aadSz], %[key], %[gcm_h], %[tmp], %[reg], %[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), + [nonce] "+r" (nonce), [nonceSz] "+r" (nonceSz), [tag] "+r" (tag), + [tagSz] "+r" (tagSz), [aad] "+r" (aad), [aadSz] "+r" (aadSz), + [key] "+r" (key), [gcm_h] "+r" (gcm_h), [tmp] "+r" (tmp), + [reg] "+r" (reg), [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "uxtb lr, r10, ror #16\n\t" -#endif + : + : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [nonce] "r" (nonce), + [nonceSz] "r" (nonceSz), [tag] "r" (tag), [tagSz] "r" (tagSz), + [aad] "r" (aad), [aadSz] "r" (aadSz), [key] "r" (key), + [gcm_h] "r" (gcm_h), [tmp] "r" (tmp), [reg] "r" (reg), [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", + "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#ifdef HAVE_AES_DECRYPT +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER int AES_GCM_decrypt_AARCH32(const byte* in_p, byte* out_p, + word32 sz_p, const byte* nonce_p, word32 nonceSz_p, const byte* tag_p, + word32 tagSz_p, const byte* aad_p, word32 aadSz_p, byte* key_p, + byte* gcm_h_p, byte* tmp_p, byte* reg_p, int nr_p) #else - "ubfx lr, r10, #16, #8\n\t" -#endif - "eor r4, r4, r2, lsl #24\n\t" - "lsr r2, r9, #24\n\t" - "ldrb r5, [r0, r5, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r6, r9, #24\n\t" - "lsr r6, r6, #24\n\t" +WC_OMIT_FRAME_POINTER int AES_GCM_decrypt_AARCH32(const byte* in, byte* out, + word32 sz, const byte* nonce, word32 nonceSz, const byte* tag, word32 tagSz, + const byte* aad, word32 aadSz, byte* key, byte* gcm_h, byte* tmp, byte* reg, + int nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* in __asm__ ("r0") = (const byte*)in_p; + register byte* out __asm__ ("r1") = (byte*)out_p; + register word32 sz __asm__ ("r2") = (word32)sz_p; + register const byte* nonce __asm__ ("r3") = (const byte*)nonce_p; + register word32 nonceSz __asm__ ("r12") = (word32)nonceSz_p; + register const byte* tag __asm__ ("lr") = (const byte*)tag_p; + register word32 tagSz __asm__ ("r4") = (word32)tagSz_p; + register const byte* aad __asm__ ("r5") = (const byte*)aad_p; + register word32 aadSz __asm__ ("r6") = (word32)aadSz_p; + register byte* key __asm__ ("r7") = (byte*)key_p; + register byte* gcm_h __asm__ ("r8") = (byte*)gcm_h_p; + register byte* tmp __asm__ ("r9") = (byte*)tmp_p; + register byte* reg __asm__ ("r10") = (byte*)reg_p; + register int nr __asm__ ("r11") = (int)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "uxtb r6, r9\n\t" -#endif + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[tagSz], %[aad], %[aadSz], %[key], %[gcm_h], %[tmp], %[reg], %[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" + "push {%[nonceSz], %[tag]}\n\t" + /* key */ + "ldr r7, [sp, #20]\n\t" + /* tmp */ + "ldr r9, [sp, #28]\n\t" + /* aadSz */ + "ldr r6, [sp, #16]\n\t" + /* gcm_h */ + "ldr r8, [sp, #24]\n\t" + "vmov.i8 q13, #0x87\n\t" + "veor.8 q7, q7, q7\n\t" + "vshr.u64 q13, q13, #56\n\t" + "vld1.32 {q8}, [r8]\n\t" + "orr r10, r6, %[sz]\n\t" + "cmp r10, #32\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_h_done_%=\n\t" + /* Square H => H^2 */ + "vmull.p64 q1, d17, d17\n\t" + "vmull.p64 q0, d16, d16\n\t" + "vmull.p64 q9, d3, d27\n\t" + "veor.8 d2, d2, d19\n\t" + "veor.8 d1, d1, d18\n\t" + "vmull.p64 q9, d2, d26\n\t" + "veor.8 q9, q9, q0\n\t" + "cmp r10, #0x40\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_h_done_%=\n\t" + /* Multiply H and H^2 => H^3 */ + "vmull.p64 q2, d17, d18\n\t" + "vmull.p64 q3, d16, d19\n\t" + "vmull.p64 q0, d16, d18\n\t" + "vmull.p64 q1, d17, d19\n\t" + "veor.8 q2, q2, q3\n\t" + /* Reduce */ + "vmull.p64 q10, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d21\n\t" + "veor.8 d1, d1, d20\n\t" + "vmull.p64 q10, d2, d26\n\t" + "veor.8 q10, q10, q0\n\t" + /* Square H^2 => H^4 */ + "vmull.p64 q1, d19, d19\n\t" + "vmull.p64 q0, d18, d18\n\t" + "vmull.p64 q11, d3, d27\n\t" + "veor.8 d2, d2, d23\n\t" + "veor.8 d1, d1, d22\n\t" + "vmull.p64 q11, d2, d26\n\t" + "veor.8 q11, q11, q0\n\t" + /* Done */ + "\n" + "L_aes_gcm_decrypt_arm32_crypto_h_done_%=:\n\t" + /* aad */ + "ldr r5, [sp, #12]\n\t" + "lsr r10, r6, #4\n\t" + "cmp r10, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_aad_done_%=\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_aad_start_1_%=\n\t" + "cmp r10, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_aad_start_2_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_aad_start_4_%=:\n\t" + "vldm r5!, {q0-q2}\n\t" + "vmov.i8 q12, #0x55\n\t" + "vmov.i8 q5, #51\n\t" + "vshl.u8 q14, q0, #1\n\t" + "vshl.u8 q15, q1, #1\n\t" + "vshl.u8 q4, q2, #1\n\t" + "ldr lr, [r5], #4\n\t" + "vshr.u8 q0, q0, #1\n\t" + "vshr.u8 q1, q1, #1\n\t" + "vshr.u8 q2, q2, #1\n\t" + "ldr r4, [r5], #4\n\t" + "vbit.8 q14, q0, q12\n\t" + "vbit.8 q15, q1, q12\n\t" + "vbit.8 q4, q2, q12\n\t" + "ldr r8, [r5], #4\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshl.u8 q1, q15, #2\n\t" + "vshl.u8 q2, q4, #2\n\t" + "ldr r12, [r5], #4\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vshr.u8 q15, q15, #2\n\t" + "vshr.u8 q4, q4, #2\n\t" + "rbit lr, lr\n\t" + "vbit.8 q0, q14, q5\n\t" + "rbit r4, r4\n\t" + "vbit.8 q1, q15, q5\n\t" + "rbit r8, r8\n\t" + "vbit.8 q2, q4, q5\n\t" + "rbit r12, r12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "rev lr, lr\n\t" + "vshl.u8 q15, q1, #4\n\t" + "rev r4, r4\n\t" + "vshl.u8 q4, q2, #4\n\t" + "rev r8, r8\n\t" + "vsri.u8 q14, q0, #4\n\t" + "rev r12, r12\n\t" + "vsri.u8 q15, q1, #4\n\t" + "vsri.u8 q4, q2, #4\n\t" + "vmov d10, lr, r4\n\t" + "vmov d11, r8, r12\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d11, d16\n\t" + "vmull.p64 q7, d10, d17\n\t" + "vmull.p64 q0, d10, d16\n\t" + "vmull.p64 q1, d11, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^2 */ + "vmull.p64 q3, d8, d18\n\t" + "vmull.p64 q7, d9, d19\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d9, d18\n\t" + "vmull.p64 q7, d8, d19\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^3 */ + "vmull.p64 q3, d30, d20\n\t" + "vmull.p64 q7, d31, d21\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d31, d20\n\t" + "vmull.p64 q7, d30, d21\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^4 */ + "vmull.p64 q3, d24, d22\n\t" + "vmull.p64 q7, d25, d23\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d25, d22\n\t" + "vmull.p64 q7, d24, d23\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "sub r10, r10, #4\n\t" + "cmp r10, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_aad_start_4_%=\n\t" + "cmp r10, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_aad_done_%=\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_aad_start_1_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_aad_start_2_%=:\n\t" + "vld1.32 {q14-q15}, [r5]!\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshl.u8 q1, q15, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vshr.u8 q15, q15, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vbif.8 q15, q1, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshl.u8 q1, q15, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vshr.u8 q15, q15, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vbit.8 q1, q15, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vshl.u8 q15, q1, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "vsri.u8 q15, q1, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d31, d16\n\t" + "vmull.p64 q7, d30, d17\n\t" + "vmull.p64 q0, d30, d16\n\t" + "vmull.p64 q1, d31, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^2 */ + "vmull.p64 q3, d24, d18\n\t" + "vmull.p64 q7, d25, d19\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d25, d18\n\t" + "vmull.p64 q7, d24, d19\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "sub r10, r10, #2\n\t" + "cmp r10, #0\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_aad_done_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_aad_start_1_%=:\n\t" + "vld1.32 {q14}, [r5]!\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q7, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "\n" + "L_aes_gcm_decrypt_arm32_crypto_aad_done_%=:\n\t" + "ands r11, r6, #15\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_aad_partial_done_%=\n\t" + "veor.8 q0, q0, q0\n\t" + "mov r12, r11\n\t" + "vst1.32 {q0}, [r9]\n\t" + "cmp r12, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_aad_start_sw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_aad_start_dw_%=:\n\t" + "ldr r8, [r5], #4\n\t" + "sub r12, r12, #4\n\t" + "str r8, [r9], #4\n\t" + "cmp r12, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_aad_start_dw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_aad_start_sw_%=:\n\t" + "cmp r12, #2\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_aad_start_byte_%=\n\t" + "ldrh r8, [r5], #2\n\t" + "sub r12, r12, #2\n\t" + "strh r8, [r9], #2\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_aad_start_byte_%=:\n\t" + "cmp r12, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_aad_end_bytes_%=\n\t" + "ldrb r8, [r5], #1\n\t" + "subs r12, r12, #1\n\t" + "strb r8, [r9], #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_aad_start_byte_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_aad_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "vld1.32 {q14}, [r9]\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q7, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "\n" + "L_aes_gcm_decrypt_arm32_crypto_aad_partial_done_%=:\n\t" + /* in */ + "lsr r10, %[sz], #4\n\t" + "cmp r10, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_in_done_%=\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_in_start_1_%=\n\t" + "cmp r10, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_in_start_2_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_in_start_4_%=:\n\t" + "vldm %[in]!, {q0-q2}\n\t" + "vmov.i8 q12, #0x55\n\t" + "vmov.i8 q5, #51\n\t" + "vshl.u8 q14, q0, #1\n\t" + "vshl.u8 q15, q1, #1\n\t" + "vshl.u8 q4, q2, #1\n\t" + "ldr lr, [%[in]], #4\n\t" + "vshr.u8 q0, q0, #1\n\t" + "vshr.u8 q1, q1, #1\n\t" + "vshr.u8 q2, q2, #1\n\t" + "ldr r4, [%[in]], #4\n\t" + "vbit.8 q14, q0, q12\n\t" + "vbit.8 q15, q1, q12\n\t" + "vbit.8 q4, q2, q12\n\t" + "ldr r8, [%[in]], #4\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshl.u8 q1, q15, #2\n\t" + "vshl.u8 q2, q4, #2\n\t" + "ldr r12, [%[in]], #4\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vshr.u8 q15, q15, #2\n\t" + "vshr.u8 q4, q4, #2\n\t" + "rbit lr, lr\n\t" + "vbit.8 q0, q14, q5\n\t" + "rbit r4, r4\n\t" + "vbit.8 q1, q15, q5\n\t" + "rbit r8, r8\n\t" + "vbit.8 q2, q4, q5\n\t" + "rbit r12, r12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "rev lr, lr\n\t" + "vshl.u8 q15, q1, #4\n\t" + "rev r4, r4\n\t" + "vshl.u8 q4, q2, #4\n\t" + "rev r8, r8\n\t" + "vsri.u8 q14, q0, #4\n\t" + "rev r12, r12\n\t" + "vsri.u8 q15, q1, #4\n\t" + "vsri.u8 q4, q2, #4\n\t" + "vmov d10, lr, r4\n\t" + "vmov d11, r8, r12\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d11, d16\n\t" + "vmull.p64 q7, d10, d17\n\t" + "vmull.p64 q0, d10, d16\n\t" + "vmull.p64 q1, d11, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^2 */ + "vmull.p64 q3, d8, d18\n\t" + "vmull.p64 q7, d9, d19\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d9, d18\n\t" + "vmull.p64 q7, d8, d19\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^3 */ + "vmull.p64 q3, d30, d20\n\t" + "vmull.p64 q7, d31, d21\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d31, d20\n\t" + "vmull.p64 q7, d30, d21\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^4 */ + "vmull.p64 q3, d24, d22\n\t" + "vmull.p64 q7, d25, d23\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d25, d22\n\t" + "vmull.p64 q7, d24, d23\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "sub r10, r10, #4\n\t" + "cmp r10, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_in_start_4_%=\n\t" + "cmp r10, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_in_done_%=\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_in_start_1_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_in_start_2_%=:\n\t" + "vld1.32 {q14-q15}, [%[in]]!\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshl.u8 q1, q15, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vshr.u8 q15, q15, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vbif.8 q15, q1, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshl.u8 q1, q15, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vshr.u8 q15, q15, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vbit.8 q1, q15, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vshl.u8 q15, q1, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "vsri.u8 q15, q1, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d31, d16\n\t" + "vmull.p64 q7, d30, d17\n\t" + "vmull.p64 q0, d30, d16\n\t" + "vmull.p64 q1, d31, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* X += C * H^2 */ + "vmull.p64 q3, d24, d18\n\t" + "vmull.p64 q7, d25, d19\n\t" + "veor.8 q0, q0, q3\n\t" + "veor.8 q1, q1, q7\n\t" + "vmull.p64 q3, d25, d18\n\t" + "vmull.p64 q7, d24, d19\n\t" + "veor.8 q2, q2, q3\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "sub r10, r10, #2\n\t" + "cmp r10, #0\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_in_done_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_in_start_1_%=:\n\t" + "vld1.32 {q14}, [%[in]]!\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q7, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "\n" + "L_aes_gcm_decrypt_arm32_crypto_in_done_%=:\n\t" + "ands r11, %[sz], #15\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_in_partial_done_%=\n\t" + "veor.8 q0, q0, q0\n\t" + "mov r12, r11\n\t" + "vst1.32 {q0}, [r9]\n\t" + "cmp r12, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_in_start_sw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_in_start_dw_%=:\n\t" + "ldr r8, [%[in]], #4\n\t" + "sub r12, r12, #4\n\t" + "str r8, [r9], #4\n\t" + "cmp r12, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_in_start_dw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_in_start_sw_%=:\n\t" + "cmp r12, #2\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_in_start_byte_%=\n\t" + "ldrh r8, [%[in]], #2\n\t" + "sub r12, r12, #2\n\t" + "strh r8, [r9], #2\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_in_start_byte_%=:\n\t" + "cmp r12, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_in_end_bytes_%=\n\t" + "ldrb r8, [%[in]], #1\n\t" + "subs r12, r12, #1\n\t" + "strb r8, [r9], #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_in_start_byte_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_in_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "vld1.32 {q14}, [r9]\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q7, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q7, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q7\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + /* Done GHASH */ + "\n" + "L_aes_gcm_decrypt_arm32_crypto_in_partial_done_%=:\n\t" + "sub %[in], %[in], %[sz]\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_done_gcm_%=:\n\t" + /* nonceSz */ + "ldr r12, [sp]\n\t" + /* Load Nonce */ + "cmp r12, #12\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_ghash_nonce_%=\n\t" + "ldr r5, [%[nonce]]\n\t" + "ldr r8, [%[nonce], #4]\n\t" + "ldr r12, [%[nonce], #8]\n\t" + "vmov.i32 q6, #0x1000000\n\t" + "vmov.32 s24, r5\n\t" + "vmov.32 s25, r8\n\t" + "vmov.32 s26, r12\n\t" + "mov r5, #1\n\t" + "b L_aes_gcm_decrypt_arm32_crypto_done_nonce_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_ghash_nonce_%=:\n\t" + "lsr r10, r12, #4\n\t" + "cmp r10, #0\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_nonce_done_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_nonce_start_1_%=:\n\t" + "vld1.32 {q14}, [%[nonce]]!\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q6, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q6, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q6\n\t" + /* Reduce */ + "vmull.p64 q6, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d13\n\t" + "veor.8 d1, d1, d12\n\t" + "vmull.p64 q6, d2, d26\n\t" + "veor.8 q6, q6, q0\n\t" + /* Done GHASH */ + "subs r10, r10, #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_nonce_start_1_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_nonce_done_%=:\n\t" + "ands r11, r12, #15\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_nonce_partial_done_%=\n\t" + "veor.8 q0, q0, q0\n\t" + "mov r12, r11\n\t" + "vst1.32 {q0}, [r9]\n\t" + "cmp r12, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_nonce_start_sw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_nonce_start_dw_%=:\n\t" + "ldr r8, [%[nonce]], #4\n\t" + "sub r12, r12, #4\n\t" + "str r8, [r9], #4\n\t" + "cmp r12, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_nonce_start_dw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_nonce_start_sw_%=:\n\t" + "cmp r12, #2\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte_%=\n\t" + "ldrh r8, [%[nonce]], #2\n\t" + "sub r12, r12, #2\n\t" + "strh r8, [r9], #2\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte_%=:\n\t" + "cmp r12, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_nonce_end_bytes_%=\n\t" + "ldrb r8, [%[nonce]], #1\n\t" + "subs r12, r12, #1\n\t" + "strb r8, [r9], #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_nonce_start_byte_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_nonce_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "vld1.32 {q14}, [r9]\n\t" + "vmov.i8 q12, #0x55\n\t" + "vshl.u8 q0, q14, #1\n\t" + "vshr.u8 q14, q14, #1\n\t" + "vbif.8 q14, q0, q12\n\t" + "vmov.i8 q12, #51\n\t" + "vshl.u8 q0, q14, #2\n\t" + "vshr.u8 q14, q14, #2\n\t" + "vbit.8 q0, q14, q12\n\t" + "vshl.u8 q14, q0, #4\n\t" + "vsri.u8 q14, q0, #4\n\t" + "veor.8 q12, q6, q14\n\t" + /* X = C * H^1 */ + "vmull.p64 q2, d25, d16\n\t" + "vmull.p64 q6, d24, d17\n\t" + "vmull.p64 q0, d24, d16\n\t" + "vmull.p64 q1, d25, d17\n\t" + "veor.8 q2, q2, q6\n\t" + /* Reduce */ + "vmull.p64 q6, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d13\n\t" + "veor.8 d1, d1, d12\n\t" + "vmull.p64 q6, d2, d26\n\t" + "veor.8 q6, q6, q0\n\t" + /* Done GHASH */ + "\n" + "L_aes_gcm_decrypt_arm32_crypto_nonce_partial_done_%=:\n\t" + "veor.8 q0, q0, q0\n\t" + /* nonceSz */ + "ldr r12, [sp]\n\t" + "lsr r10, r12, #29\n\t" + "lsl r11, r12, #3\n\t" + "rbit r10, r10\n\t" + "rbit r11, r11\n\t" + "vmov.32 s2, r10\n\t" + "vmov.32 s3, r11\n\t" + "veor.8 q6, q6, q0\n\t" + "vmull.p64 q2, d13, d16\n\t" + "vmull.p64 q12, d12, d17\n\t" + "vmull.p64 q0, d12, d16\n\t" + "vmull.p64 q1, d13, d17\n\t" + "veor.8 q2, q2, q12\n\t" + "vmull.p64 q6, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d13\n\t" + "veor.8 d1, d1, d12\n\t" + "vmull.p64 q6, d2, d26\n\t" + "veor.8 q6, q6, q0\n\t" + "vmov.32 r5, s24\n\t" + "vmov.32 r8, s25\n\t" + "vmov.32 r12, s26\n\t" + "rbit r5, r5\n\t" + "rbit r8, r8\n\t" + "rbit r12, r12\n\t" + "rev r5, r5\n\t" + "rev r8, r8\n\t" + "rev r12, r12\n\t" + "vmov.32 s24, r5\n\t" + "vmov.32 s25, r8\n\t" + "vmov.32 s26, r12\n\t" + "vmov.32 r5, s27\n\t" + "rbit r5, r5\n\t" + "rev r5, r5\n\t" + "vmov.32 s27, r5\n\t" + "rev r5, r5\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_done_nonce_%=:\n\t" + /* reg */ + "ldr r9, [sp, #32]\n\t" + "vst1.32 {q7}, [r9]\n\t" + /* tmp */ + "ldr r9, [sp, #28]\n\t" + "vldm.32 r7!, {q0-q3}\n\t" + "vldm.32 r7!, {q7-q13}\n\t" + /* nr */ + "ldr r12, [sp, #36]\n\t" + "lsr r10, %[sz], #4\n\t" + "and r11, %[sz], #15\n\t" + "cmp r12, #12\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_start_128_%=\n\t" + "bgt L_aes_gcm_decrypt_arm32_crypto_start_256_%=\n\t" + /* AES_GCM_192 */ +#ifndef NO_AES_192 + "cmp r10, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_192_done_%=\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_192_start_1_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_start_2_%=:\n\t" + "add r8, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "add r5, r5, #2\n\t" + "vmov.8 q5, q6\n\t" + "rev r8, r8\n\t" + "rev r12, r5\n\t" + "vmov s19, r8\n\t" + "vmov s23, r12\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q0\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q1\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q2\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q3\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q7\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q8\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q9\n\t" + "aesmc.8 q5, q5\n\t" + "subs r10, r10, #2\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q10\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q11\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q12\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q13\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "aese.8 q5, q14\n\t" + "veor.8 q5, q5, q15\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "sub r7, r7, #16\n\t" + "veor.8 q14, q14, q4\n\t" + "veor.8 q15, q15, q5\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "cmp r10, #1\n\t" + "bgt L_aes_gcm_decrypt_arm32_crypto_192_start_2_%=\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_192_done_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_start_1_%=:\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "sub r7, r7, #16\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [%[out]]!\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_done_%=:\n\t" + "ands r11, %[sz], #15\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_192_partial_done_%=\n\t" + "veor.8 q14, q14, q14\n\t" + "mov r4, r11\n\t" + "vst1.32 {q14}, [r9]\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_192_start_sw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_start_dw_%=:\n\t" + "ldr lr, [%[in]], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [r9], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_192_start_dw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_192_start_byte_%=\n\t" + "ldrh lr, [%[in]], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [r9], #2\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_192_end_bytes_%=\n\t" + "ldrb lr, [%[in]], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [r9], #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_192_start_byte_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "vld1.8 {q14}, [r9]\n\t" + "sub r7, r7, #16\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [r9]\n\t" + "mov r4, r11\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_192_out_start_sw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_out_start_dw_%=:\n\t" + "ldr lr, [r9], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [%[out]], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_192_out_start_dw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_out_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte_%=\n\t" + "ldrh lr, [r9], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [%[out]], #2\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_192_out_end_bytes_%=\n\t" + "ldrb lr, [r9], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [%[out]], #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_192_out_start_byte_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_out_end_bytes_%=:\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_192_partial_done_%=:\n\t" + /* Finish */ + "add r8, %[sz], #15\n\t" + "sub r8, r5, r8, lsr #4\n\t" + "rev r8, r8\n\t" + "vmov.32 s27, r8\n\t" + "aese.8 q6, q0\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q1\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q2\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q3\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q7\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q8\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q9\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q10\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q11\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q12\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q6, q13\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q6, q14\n\t" + "veor.8 q6, q6, q15\n\t" + "sub r7, r7, #16\n\t" +#endif /* !NO_AES_192 */ + "b L_aes_gcm_decrypt_arm32_crypto_done_enc_%=\n\t" + /* AES_GCM_256 */ + "\n" + "L_aes_gcm_decrypt_arm32_crypto_start_256_%=:\n\t" +#ifndef NO_AES_256 + "cmp r10, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_256_done_%=\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_256_start_1_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_start_2_%=:\n\t" + "add r8, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "add r5, r5, #2\n\t" + "vmov.8 q5, q6\n\t" + "rev r8, r8\n\t" + "rev r12, r5\n\t" + "vmov s19, r8\n\t" + "vmov s23, r12\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q0\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q1\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q2\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q3\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q7\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q8\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q9\n\t" + "aesmc.8 q5, q5\n\t" + "subs r10, r10, #2\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q10\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q11\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q12\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q13\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q15}, [r7]!\n\t" + "aese.8 q4, q14\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q14\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q15\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q15\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "aese.8 q5, q14\n\t" + "veor.8 q5, q5, q15\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "sub r7, r7, #48\n\t" + "veor.8 q14, q14, q4\n\t" + "veor.8 q15, q15, q5\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "cmp r10, #1\n\t" + "bgt L_aes_gcm_decrypt_arm32_crypto_256_start_2_%=\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_256_done_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_start_1_%=:\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]!\n\t" + "aese.8 q4, q14\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q15\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "sub r7, r7, #48\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [%[out]]!\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_done_%=:\n\t" + "ands r11, %[sz], #15\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_256_partial_done_%=\n\t" + "veor.8 q14, q14, q14\n\t" + "mov r4, r11\n\t" + "vst1.32 {q14}, [r9]\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_256_start_sw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_start_dw_%=:\n\t" + "ldr lr, [%[in]], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [r9], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_256_start_dw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_256_start_byte_%=\n\t" + "ldrh lr, [%[in]], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [r9], #2\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_256_end_bytes_%=\n\t" + "ldrb lr, [%[in]], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [r9], #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_256_start_byte_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q12\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q13\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]!\n\t" + "aese.8 q4, q14\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q4, q15\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q4, q14\n\t" + "veor.8 q4, q4, q15\n\t" + "vld1.8 {q14}, [r9]\n\t" + "sub r7, r7, #48\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [r9]\n\t" + "mov r4, r11\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_256_out_start_sw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_out_start_dw_%=:\n\t" + "ldr lr, [r9], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [%[out]], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_256_out_start_dw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_out_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte_%=\n\t" + "ldrh lr, [r9], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [%[out]], #2\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_256_out_end_bytes_%=\n\t" + "ldrb lr, [r9], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [%[out]], #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_256_out_start_byte_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_out_end_bytes_%=:\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_256_partial_done_%=:\n\t" + /* Finish */ + "add r8, %[sz], #15\n\t" + "sub r8, r5, r8, lsr #4\n\t" + "rev r8, r8\n\t" + "vmov.32 s27, r8\n\t" + "aese.8 q6, q0\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q1\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q2\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q3\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q7\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q8\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q9\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q10\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q11\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q12\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q6, q13\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q15}, [r7]!\n\t" + "aese.8 q6, q14\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q14}, [r7]!\n\t" + "aese.8 q6, q15\n\t" + "aesmc.8 q6, q6\n\t" + "vld1.32 {q15}, [r7]\n\t" + "aese.8 q6, q14\n\t" + "veor.8 q6, q6, q15\n\t" + "sub r7, r7, #48\n\t" +#endif /* !NO_AES_256 */ + "b L_aes_gcm_decrypt_arm32_crypto_done_enc_%=\n\t" + /* AES_GCM_128 */ + "\n" + "L_aes_gcm_decrypt_arm32_crypto_start_128_%=:\n\t" +#ifndef NO_AES_128 + "cmp r10, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_128_done_%=\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_128_start_1_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_start_2_%=:\n\t" + "add r8, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "add r5, r5, #2\n\t" + "vmov.8 q5, q6\n\t" + "rev r8, r8\n\t" + "rev r12, r5\n\t" + "vmov s19, r8\n\t" + "vmov s23, r12\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q0\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q1\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q2\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q3\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q7\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q8\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q9\n\t" + "aesmc.8 q5, q5\n\t" + "subs r10, r10, #2\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q10\n\t" + "aesmc.8 q5, q5\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q5, q11\n\t" + "aesmc.8 q5, q5\n\t" + "vld1.8 {q14-q15}, [%[in]]!\n\t" + "aese.8 q4, q12\n\t" + "veor.8 q4, q4, q13\n\t" + "aese.8 q5, q12\n\t" + "veor.8 q5, q5, q13\n\t" + "veor.8 q14, q14, q4\n\t" + "veor.8 q15, q15, q5\n\t" + "vst1.8 {q14-q15}, [%[out]]!\n\t" + "cmp r10, #1\n\t" + "bgt L_aes_gcm_decrypt_arm32_crypto_128_start_2_%=\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_128_done_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_start_1_%=:\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.8 {q14}, [%[in]]!\n\t" + "aese.8 q4, q12\n\t" + "veor.8 q4, q4, q13\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [%[out]]!\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_done_%=:\n\t" + "ands r11, %[sz], #15\n\t" + "beq L_aes_gcm_decrypt_arm32_crypto_128_partial_done_%=\n\t" + "veor.8 q14, q14, q14\n\t" + "mov r4, r11\n\t" + "vst1.32 {q14}, [r9]\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_128_start_sw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_start_dw_%=:\n\t" + "ldr lr, [%[in]], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [r9], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_128_start_dw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_128_start_byte_%=\n\t" + "ldrh lr, [%[in]], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [r9], #2\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_128_end_bytes_%=\n\t" + "ldrb lr, [%[in]], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [r9], #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_128_start_byte_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_end_bytes_%=:\n\t" + "sub r9, r9, r11\n\t" + "add r5, r5, #1\n\t" + "vmov.8 q4, q6\n\t" + "rev r8, r5\n\t" + "vmov s19, r8\n\t" + "aese.8 q4, q0\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q1\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q2\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q3\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q7\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q8\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q9\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q10\n\t" + "aesmc.8 q4, q4\n\t" + "aese.8 q4, q11\n\t" + "aesmc.8 q4, q4\n\t" + "vld1.8 {q14}, [r9]\n\t" + "aese.8 q4, q12\n\t" + "veor.8 q4, q4, q13\n\t" + "veor.8 q14, q14, q4\n\t" + "vst1.32 {q14}, [r9]\n\t" + "mov r4, r11\n\t" + "cmp r4, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_128_out_start_sw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_out_start_dw_%=:\n\t" + "ldr lr, [r9], #4\n\t" + "sub r4, r4, #4\n\t" + "str lr, [%[out]], #4\n\t" + "cmp r4, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_128_out_start_dw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_out_start_sw_%=:\n\t" + "cmp r4, #2\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte_%=\n\t" + "ldrh lr, [r9], #2\n\t" + "sub r4, r4, #2\n\t" + "strh lr, [%[out]], #2\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte_%=:\n\t" + "cmp r4, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_128_out_end_bytes_%=\n\t" + "ldrb lr, [r9], #1\n\t" + "subs r4, r4, #1\n\t" + "strb lr, [%[out]], #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_128_out_start_byte_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_out_end_bytes_%=:\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_128_partial_done_%=:\n\t" + /* Finish */ + "add r8, %[sz], #15\n\t" + "sub r8, r5, r8, lsr #4\n\t" + "rev r8, r8\n\t" + "vmov.32 s27, r8\n\t" + "aese.8 q6, q0\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q1\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q2\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q3\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q7\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q8\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q9\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q10\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q11\n\t" + "aesmc.8 q6, q6\n\t" + "aese.8 q6, q12\n\t" + "veor.8 q6, q6, q13\n\t" +#endif /* !NO_AES_128 */ + "\n" + "L_aes_gcm_decrypt_arm32_crypto_done_enc_%=:\n\t" + "vmov.i8 q13, #0x87\n\t" + "vshr.u64 q13, q13, #56\n\t" + /* gcm_h */ + "ldr r8, [sp, #24]\n\t" + "vld1.32 {q8}, [r8]\n\t" + /* reg */ + "ldr r9, [sp, #32]\n\t" + "vld1.32 {q7}, [r9]\n\t" + /* tmp */ + "ldr r9, [sp, #28]\n\t" + "lsr lr, r6, #29\n\t" + "lsl r6, r6, #3\n\t" + "rbit lr, lr\n\t" + "rbit r6, r6\n\t" + "vmov s0, lr\n\t" + "vmov s1, r6\n\t" + "lsr lr, %[sz], #29\n\t" + "lsl %[sz], %[sz], #3\n\t" + "rbit lr, lr\n\t" + "rbit %[sz], %[sz]\n\t" + "vmov s2, lr\n\t" + "vmov s3, %[sz]\n\t" + "veor.8 q7, q7, q0\n\t" + "vmull.p64 q2, d15, d16\n\t" + "vmull.p64 q3, d14, d17\n\t" + "vmull.p64 q0, d14, d16\n\t" + "vmull.p64 q1, d15, d17\n\t" + "veor.8 q2, q2, q3\n\t" + /* Reduce */ + "vmull.p64 q7, d3, d27\n\t" + "veor.8 d2, d2, d5\n\t" + "veor.8 d1, d1, d4\n\t" + "veor.8 d2, d2, d15\n\t" + "veor.8 d1, d1, d14\n\t" + "vmull.p64 q7, d2, d26\n\t" + "veor.8 q7, q7, q0\n\t" + "vmov.i8 q0, #0x55\n\t" + "vshl.u8 q1, q7, #1\n\t" + "vshr.u8 q7, q7, #1\n\t" + "vbif.8 q7, q1, q0\n\t" + "vmov.i8 q0, #51\n\t" + "vshl.u8 q1, q7, #2\n\t" + "vshr.u8 q7, q7, #2\n\t" + "vbit.8 q1, q7, q0\n\t" + "vshl.u8 q7, q1, #4\n\t" + "vsri.u8 q7, q1, #4\n\t" + /* DONE */ + "veor.8 q7, q7, q6\n\t" + /* tag */ + "ldr lr, [sp, #4]\n\t" + /* tagSz */ + "ldr r4, [sp, #8]\n\t" + "cmp r4, #16\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_tag_part_tag_%=\n\t" + "vld1.8 {q0}, [lr]\n\t" + "b L_aes_gcm_decrypt_arm32_crypto_tag_tag_loaded_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_tag_part_tag_%=:\n\t" + "veor.8 q0, q0, q0\n\t" + "mov r12, r4\n\t" + "vst1.32 {q0}, [r9]\n\t" + "cmp r12, #4\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_sw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_dw_%=:\n\t" + "ldr r8, [lr], #4\n\t" + "sub r12, r12, #4\n\t" + "str r8, [r9], #4\n\t" + "cmp r12, #4\n\t" + "bge L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_dw_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_sw_%=:\n\t" + "cmp r12, #2\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte_%=\n\t" + "ldrh r8, [lr], #2\n\t" + "sub r12, r12, #2\n\t" + "strh r8, [r9], #2\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte_%=:\n\t" + "cmp r12, #1\n\t" + "blt L_aes_gcm_decrypt_arm32_crypto_tag_tag_end_bytes_%=\n\t" + "ldrb r8, [lr], #1\n\t" + "subs r12, r12, #1\n\t" + "strb r8, [r9], #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_tag_tag_start_byte_%=\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_tag_tag_end_bytes_%=:\n\t" + "sub r9, r9, r4\n\t" + "vld1.32 {q0}, [r9]\n\t" + "mov r12, #16\n\t" + "vst1.32 {q7}, [r9]\n\t" + "sub r12, r12, r4\n\t" + "eor r8, r8, r8\n\t" + "add r9, r9, r4\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_tag_calc_tag_byte_%=:\n\t" + "strb r8, [r9], #1\n\t" + "subs r12, r12, #1\n\t" + "bne L_aes_gcm_decrypt_arm32_crypto_tag_calc_tag_byte_%=\n\t" + "subs r9, r9, #16\n\t" + "vld1.32 {q7}, [r9]\n\t" + "\n" + "L_aes_gcm_decrypt_arm32_crypto_tag_tag_loaded_%=:\n\t" + "vceq.i32 q0, q0, q7\n\t" + "vmov r5, s0\n\t" + "vmov r8, s1\n\t" + "and r12, r5, r8\n\t" + "vmov r5, s2\n\t" + "vmov r8, s3\n\t" + "and r12, r12, r5\n\t" + "and r12, r12, r8\n\t" + "mov r5, #-180\n\t" + "mvn r12, r12\n\t" + "and %[in], r5, r12\n\t" + "pop {%[nonceSz], %[tag]}\n\t" + "pop {%[tagSz], %[aad], %[aadSz], %[key], %[gcm_h], %[tmp], %[reg], %[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), + [nonce] "+r" (nonce), [nonceSz] "+r" (nonceSz), [tag] "+r" (tag), + [tagSz] "+r" (tagSz), [aad] "+r" (aad), [aadSz] "+r" (aadSz), + [key] "+r" (key), [gcm_h] "+r" (gcm_h), [tmp] "+r" (tmp), + [reg] "+r" (reg), [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "ubfx r6, r9, #0, #8\n\t" -#endif - "eor r5, r5, r7, lsl #8\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r8, #16\n\t" - "lsr r7, r7, #24\n\t" + : + : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [nonce] "r" (nonce), + [nonceSz] "r" (nonceSz), [tag] "r" (tag), [tagSz] "r" (tagSz), + [aad] "r" (aad), [aadSz] "r" (aadSz), [key] "r" (key), + [gcm_h] "r" (gcm_h), [tmp] "r" (tmp), [reg] "r" (reg), [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", + "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); + return (word32)(size_t)in; +} + +#endif /* HAVE_AES_DECRYPT */ +#endif /* HAVE_AESGCM */ +#ifdef WOLFSSL_AES_XTS +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_XTS_encrypt_AARCH32(const byte* in_p, + byte* out_p, word32 sz_p, const byte* i_p, byte* key_p, byte* key2_p, + byte* tmp_p, int nr_p) #else - "uxtb r7, r8, ror #8\n\t" -#endif +WC_OMIT_FRAME_POINTER void AES_XTS_encrypt_AARCH32(const byte* in, byte* out, + word32 sz, const byte* i, byte* key, byte* key2, byte* tmp, int nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* in __asm__ ("r0") = (const byte*)in_p; + register byte* out __asm__ ("r1") = (byte*)out_p; + register word32 sz __asm__ ("r2") = (word32)sz_p; + register const byte* i __asm__ ("r3") = (const byte*)i_p; + register byte* key __asm__ ("r12") = (byte*)key_p; + register byte* key2 __asm__ ("lr") = (byte*)key2_p; + register byte* tmp __asm__ ("r4") = (byte*)tmp_p; + register int nr __asm__ ("r5") = (int)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r6") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "ubfx r7, r8, #8, #8\n\t" -#endif - "eor r5, r5, lr, lsl #16\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r11, #8\n\t" - "lsr lr, lr, #24\n\t" + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[tmp], %[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" + "push {%[key], %[key2]}\n\t" + "ldr r12, [sp]\n\t" + "ldr lr, [sp, #4]\n\t" + "ldr r5, [sp, #12]\n\t" + "vldm.32 lr!, {q2-q9}\n\t" + "lsr r4, %[sz], #4\n\t" + "and %[sz], %[sz], #15\n\t" + "vld1.8 {q0}, [%[i]]\n\t" + "cmp r5, #12\n\t" + "blt L_aes_xts_encrypt_arm32_crypto_start_128_%=\n\t" + "bgt L_aes_xts_encrypt_arm32_crypto_start_256_%=\n\t" + /* AES_XTS_192 */ +#ifndef NO_AES_192 + "vldm.32 lr!, {q10-q14}\n\t" + "aese.8 q0, q2\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q11\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q12\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q13\n\t" + "veor.32 q0, q0, q14\n\t" + "vmov r6, r7, d0\n\t" + "vmov r8, r9, d1\n\t" + "vldm.32 r12!, {q2-q9}\n\t" + "vldm.32 r12!, {q10-q14}\n\t" + "mov lr, #0x87\n\t" + "cmp r4, #1\n\t" + "blt L_aes_xts_encrypt_arm32_crypto_192_done_%=\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_192_start_1_%=:\n\t" + "vld1.8 {q1}, [%[in]]!\n\t" + "veor.32 q1, q1, q0\n\t" + "aese.8 q1, q2\n\t" + "aesmc.8 q1, q1\n\t" + "and r5, lr, r9, asr #31\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "lsl r9, r9, #1\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "orr r9, r9, r8, lsr #31\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "lsl r8, r8, #1\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "orr r8, r8, r7, lsr #31\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "lsl r7, r7, #1\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "orr r7, r7, r6, lsr #31\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "eor r6, r5, r6, lsl #1\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q11\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q12\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q13\n\t" + "veor.32 q1, q1, q14\n\t" + "veor.32 q1, q1, q0\n\t" + "vmov d0, r6, r7\n\t" + "vmov d1, r8, r9\n\t" + "subs r4, r4, #1\n\t" + "vst1.8 {q1}, [%[out]]!\n\t" + "bne L_aes_xts_encrypt_arm32_crypto_192_start_1_%=\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_192_done_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_xts_encrypt_arm32_crypto_192_partial_done_%=\n\t" + "sub %[out], %[out], #16\n\t" + "ldr r4, [sp, #8]\n\t" + "vld1.8 {q1}, [%[out]]!\n\t" + "vst1.32 {q1}, [r4]\n\t" + "mov r5, %[sz]\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_192_start_byte_%=:\n\t" + "ldrb r8, [r4]\n\t" + "ldrb r9, [%[in]], #1\n\t" + "strb r8, [%[out]], #1\n\t" + "strb r9, [r4], #1\n\t" + "subs r5, r5, #1\n\t" + "bgt L_aes_xts_encrypt_arm32_crypto_192_start_byte_%=\n\t" + "sub %[out], %[out], %[sz]\n\t" + "sub r4, r4, %[sz]\n\t" + "sub %[out], %[out], #16\n\t" + "vld1.32 {q1}, [r4]\n\t" + "veor.32 q1, q1, q0\n\t" + "aese.8 q1, q2\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q11\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q12\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q13\n\t" + "veor.32 q1, q1, q14\n\t" + "veor.32 q1, q1, q0\n\t" + "vst1.8 {q1}, [%[out]]\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_192_partial_done_%=:\n\t" +#endif /* !NO_AES_192 */ + "b L_aes_xts_encrypt_arm32_crypto_done_%=\n\t" + /* AES_XTS_256 */ + "\n" + "L_aes_xts_encrypt_arm32_crypto_start_256_%=:\n\t" +#ifndef NO_AES_256 + "vldm.32 lr!, {q10-q13}\n\t" + "aese.8 q0, q2\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q11\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q12\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q14}, [lr]!\n\t" + "aese.8 q0, q13\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q15}, [lr]!\n\t" + "aese.8 q0, q14\n\t" + "aesmc.8 q0, q0\n\t" + "vld1.32 {q14}, [lr]!\n\t" + "aese.8 q0, q15\n\t" + "veor.32 q0, q0, q14\n\t" + "vmov r6, r7, d0\n\t" + "vmov r8, r9, d1\n\t" + "vldm.32 r12!, {q2-q9}\n\t" + "vldm.32 r12!, {q10-q13}\n\t" + "mov lr, #0x87\n\t" + "cmp r4, #1\n\t" + "blt L_aes_xts_encrypt_arm32_crypto_256_done_%=\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_256_start_1_%=:\n\t" + "vld1.8 {q1}, [%[in]]!\n\t" + "veor.32 q1, q1, q0\n\t" + "aese.8 q1, q2\n\t" + "aesmc.8 q1, q1\n\t" + "and r5, lr, r9, asr #31\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "lsl r9, r9, #1\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "orr r9, r9, r8, lsr #31\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "lsl r8, r8, #1\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "orr r8, r8, r7, lsr #31\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "lsl r7, r7, #1\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "orr r7, r7, r6, lsr #31\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "eor r6, r5, r6, lsl #1\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q11\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q12\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aese.8 q1, q13\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q15}, [r12]!\n\t" + "aese.8 q1, q14\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aese.8 q1, q15\n\t" + "veor.32 q1, q1, q14\n\t" + "sub r12, r12, #48\n\t" + "veor.32 q1, q1, q0\n\t" + "vmov d0, r6, r7\n\t" + "vmov d1, r8, r9\n\t" + "subs r4, r4, #1\n\t" + "vst1.8 {q1}, [%[out]]!\n\t" + "bne L_aes_xts_encrypt_arm32_crypto_256_start_1_%=\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_256_done_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_xts_encrypt_arm32_crypto_256_partial_done_%=\n\t" + "sub %[out], %[out], #16\n\t" + "ldr r4, [sp, #8]\n\t" + "vld1.8 {q1}, [%[out]]!\n\t" + "vst1.32 {q1}, [r4]\n\t" + "mov r5, %[sz]\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_256_start_byte_%=:\n\t" + "ldrb r8, [r4]\n\t" + "ldrb r9, [%[in]], #1\n\t" + "strb r8, [%[out]], #1\n\t" + "strb r9, [r4], #1\n\t" + "subs r5, r5, #1\n\t" + "bgt L_aes_xts_encrypt_arm32_crypto_256_start_byte_%=\n\t" + "sub %[out], %[out], %[sz]\n\t" + "sub r4, r4, %[sz]\n\t" + "sub %[out], %[out], #16\n\t" + "vld1.32 {q1}, [r4]\n\t" + "veor.32 q1, q1, q0\n\t" + "aese.8 q1, q2\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q11\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q12\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aese.8 q1, q13\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q15}, [r12]!\n\t" + "aese.8 q1, q14\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aese.8 q1, q15\n\t" + "veor.32 q1, q1, q14\n\t" + "veor.32 q1, q1, q0\n\t" + "vst1.8 {q1}, [%[out]]\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_256_partial_done_%=:\n\t" +#endif /* !NO_AES_256 */ + "b L_aes_xts_encrypt_arm32_crypto_done_%=\n\t" + /* AES_XTS_128 */ + "\n" + "L_aes_xts_encrypt_arm32_crypto_start_128_%=:\n\t" +#ifndef NO_AES_128 + "vldm.32 lr!, {q10-q12}\n\t" + "aese.8 q0, q2\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q3\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q4\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q5\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q6\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q7\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q8\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q9\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q10\n\t" + "aesmc.8 q0, q0\n\t" + "aese.8 q0, q11\n\t" + "veor.32 q0, q0, q12\n\t" + "vmov r6, r7, d0\n\t" + "vmov r8, r9, d1\n\t" + "vldm.32 r12!, {q2-q9}\n\t" + "vldm.32 r12!, {q10-q12}\n\t" + "mov lr, #0x87\n\t" + "cmp r4, #1\n\t" + "blt L_aes_xts_encrypt_arm32_crypto_128_done_%=\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_128_start_1_%=:\n\t" + "vld1.8 {q1}, [%[in]]!\n\t" + "veor.32 q1, q1, q0\n\t" + "aese.8 q1, q2\n\t" + "aesmc.8 q1, q1\n\t" + "and r5, lr, r9, asr #31\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "lsl r9, r9, #1\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "orr r9, r9, r8, lsr #31\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "lsl r8, r8, #1\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "orr r8, r8, r7, lsr #31\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "lsl r7, r7, #1\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "orr r7, r7, r6, lsr #31\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "eor r6, r5, r6, lsl #1\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q11\n\t" + "veor.32 q1, q1, q12\n\t" + "veor.32 q1, q1, q0\n\t" + "vmov d0, r6, r7\n\t" + "vmov d1, r8, r9\n\t" + "subs r4, r4, #1\n\t" + "vst1.8 {q1}, [%[out]]!\n\t" + "bne L_aes_xts_encrypt_arm32_crypto_128_start_1_%=\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_128_done_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_xts_encrypt_arm32_crypto_128_partial_done_%=\n\t" + "sub %[out], %[out], #16\n\t" + "ldr r4, [sp, #8]\n\t" + "vld1.8 {q1}, [%[out]]!\n\t" + "vst1.32 {q1}, [r4]\n\t" + "mov r5, %[sz]\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_128_start_byte_%=:\n\t" + "ldrb r8, [r4]\n\t" + "ldrb r9, [%[in]], #1\n\t" + "strb r8, [%[out]], #1\n\t" + "strb r9, [r4], #1\n\t" + "subs r5, r5, #1\n\t" + "bgt L_aes_xts_encrypt_arm32_crypto_128_start_byte_%=\n\t" + "sub %[out], %[out], %[sz]\n\t" + "sub r4, r4, %[sz]\n\t" + "sub %[out], %[out], #16\n\t" + "vld1.32 {q1}, [r4]\n\t" + "veor.32 q1, q1, q0\n\t" + "aese.8 q1, q2\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q11\n\t" + "veor.32 q1, q1, q12\n\t" + "veor.32 q1, q1, q0\n\t" + "vst1.8 {q1}, [%[out]]\n\t" + "\n" + "L_aes_xts_encrypt_arm32_crypto_128_partial_done_%=:\n\t" +#endif /* !NO_AES_128 */ + "\n" + "L_aes_xts_encrypt_arm32_crypto_done_%=:\n\t" + "pop {%[key], %[key2]}\n\t" + "pop {%[tmp], %[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [i] "+r" (i), + [key] "+r" (key), [key2] "+r" (key2), [tmp] "+r" (tmp), + [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "uxtb lr, r11, ror #16\n\t" -#endif + : + : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [i] "r" (i), + [key] "r" (key), [key2] "r" (key2), [tmp] "r" (tmp), [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r7", "r8", "r9", "q0", "q1", "q2", "q3", "q4", "q5", + "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#ifdef HAVE_AES_DECRYPT +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_XTS_decrypt_AARCH32(const byte* in_p, + byte* out_p, word32 sz_p, const byte* i_p, byte* key_p, byte* key2_p, + byte* tmp_p, int nr_p) #else - "ubfx lr, r11, #16, #8\n\t" -#endif - "eor r5, r5, r2, lsl #24\n\t" - "lsr r2, r10, #24\n\t" - "ldrb r6, [r0, r6, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" - "lsr r11, r11, #24\n\t" - "eor r6, r6, r7, lsl #8\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r7, r10, #24\n\t" - "lsr r7, r7, #24\n\t" +WC_OMIT_FRAME_POINTER void AES_XTS_decrypt_AARCH32(const byte* in, byte* out, + word32 sz, const byte* i, byte* key, byte* key2, byte* tmp, int nr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* in __asm__ ("r0") = (const byte*)in_p; + register byte* out __asm__ ("r1") = (byte*)out_p; + register word32 sz __asm__ ("r2") = (word32)sz_p; + register const byte* i __asm__ ("r3") = (const byte*)i_p; + register byte* key __asm__ ("r12") = (byte*)key_p; + register byte* key2 __asm__ ("lr") = (byte*)key2_p; + register byte* tmp __asm__ ("r4") = (byte*)tmp_p; + register int nr __asm__ ("r5") = (int)nr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r6") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "uxtb r7, r10\n\t" -#endif + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[tmp], %[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" + "push {%[key], %[key2]}\n\t" + "ldr r12, [sp]\n\t" + "ldr lr, [sp, #4]\n\t" + "vldm.32 lr!, {q2-q9}\n\t" + "eor r5, r5, r5\n\t" + "lsr r4, %[sz], #4\n\t" + "ands %[sz], %[sz], #15\n\t" + "sub r5, r5, %[sz]\n\t" + "sub r4, r4, r5, lsr #31\n\t" + "vld1.8 {q1}, [%[i]]\n\t" + "ldr r5, [sp, #12]\n\t" + "cmp r5, #12\n\t" + "blt L_aes_xts_decrypt_arm32_crypto_start_128_%=\n\t" + "bgt L_aes_xts_decrypt_arm32_crypto_start_256_%=\n\t" + /* AES_XTS_192 */ +#ifndef NO_AES_192 + "vldm.32 lr!, {q10-q14}\n\t" + "aese.8 q1, q2\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q11\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q12\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q13\n\t" + "veor.32 q1, q1, q14\n\t" + "vmov r6, r7, d2\n\t" + "vmov r8, r9, d3\n\t" + "vldm.32 r12!, {q2-q9}\n\t" + "vldm.32 r12!, {q10-q14}\n\t" + "mov lr, #0x87\n\t" + "cmp r4, #1\n\t" + "blt L_aes_xts_decrypt_arm32_crypto_192_done_%=\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_192_start_1_%=:\n\t" + "vld1.8 {q0}, [%[in]]!\n\t" + "veor.32 q0, q0, q1\n\t" + "aesd.8 q0, q2\n\t" + "aesimc.8 q0, q0\n\t" + "and r5, lr, r9, asr #31\n\t" + "aesd.8 q0, q3\n\t" + "aesimc.8 q0, q0\n\t" + "lsl r9, r9, #1\n\t" + "aesd.8 q0, q4\n\t" + "aesimc.8 q0, q0\n\t" + "orr r9, r9, r8, lsr #31\n\t" + "aesd.8 q0, q5\n\t" + "aesimc.8 q0, q0\n\t" + "lsl r8, r8, #1\n\t" + "aesd.8 q0, q6\n\t" + "aesimc.8 q0, q0\n\t" + "orr r8, r8, r7, lsr #31\n\t" + "aesd.8 q0, q7\n\t" + "aesimc.8 q0, q0\n\t" + "lsl r7, r7, #1\n\t" + "aesd.8 q0, q8\n\t" + "aesimc.8 q0, q0\n\t" + "orr r7, r7, r6, lsr #31\n\t" + "aesd.8 q0, q9\n\t" + "aesimc.8 q0, q0\n\t" + "eor r6, r5, r6, lsl #1\n\t" + "aesd.8 q0, q10\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q11\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q12\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q13\n\t" + "veor.32 q0, q0, q14\n\t" + "veor.32 q0, q0, q1\n\t" + "vmov d2, r6, r7\n\t" + "vmov d3, r8, r9\n\t" + "subs r4, r4, #1\n\t" + "vst1.8 {q0}, [%[out]]!\n\t" + "bne L_aes_xts_decrypt_arm32_crypto_192_start_1_%=\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_192_done_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_xts_decrypt_arm32_crypto_192_partial_done_%=\n\t" + "and r5, lr, r9, asr #31\n\t" + "lsl r9, r9, #1\n\t" + "orr r9, r9, r8, lsr #31\n\t" + "lsl r8, r8, #1\n\t" + "orr r8, r8, r7, lsr #31\n\t" + "lsl r7, r7, #1\n\t" + "orr r7, r7, r6, lsr #31\n\t" + "eor r6, r5, r6, lsl #1\n\t" + "vmov d30, r6, r7\n\t" + "vmov d31, r8, r9\n\t" + "vld1.8 {q0}, [%[in]]!\n\t" + "ldr r4, [sp, #8]\n\t" + "veor.32 q0, q0, q15\n\t" + "aesd.8 q0, q2\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q3\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q4\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q5\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q6\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q7\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q8\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q9\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q10\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q11\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q12\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q13\n\t" + "veor.32 q0, q0, q14\n\t" + "veor.32 q0, q0, q15\n\t" + "vst1.32 {q0}, [r4]\n\t" + "add %[out], %[out], #16\n\t" + "mov r5, %[sz]\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_192_start_byte_%=:\n\t" + "ldrb r8, [r4]\n\t" + "ldrb r9, [%[in]], #1\n\t" + "strb r8, [%[out]], #1\n\t" + "strb r9, [r4], #1\n\t" + "subs r5, r5, #1\n\t" + "bgt L_aes_xts_decrypt_arm32_crypto_192_start_byte_%=\n\t" + "sub %[out], %[out], %[sz]\n\t" + "sub r4, r4, %[sz]\n\t" + "sub %[out], %[out], #16\n\t" + "vld1.32 {q0}, [r4]\n\t" + "veor.32 q0, q0, q1\n\t" + "aesd.8 q0, q2\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q3\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q4\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q5\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q6\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q7\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q8\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q9\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q10\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q11\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q12\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q13\n\t" + "veor.32 q0, q0, q14\n\t" + "veor.32 q0, q0, q1\n\t" + "vst1.8 {q0}, [%[out]]\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_192_partial_done_%=:\n\t" +#endif /* !NO_AES_192 */ + "b L_aes_xts_decrypt_arm32_crypto_done_%=\n\t" + /* AES_XTS_256 */ + "\n" + "L_aes_xts_decrypt_arm32_crypto_start_256_%=:\n\t" +#ifndef NO_AES_256 + "vldm.32 lr!, {q10-q13}\n\t" + "aese.8 q1, q2\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q11\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q12\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q14}, [lr]!\n\t" + "aese.8 q1, q13\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q15}, [lr]!\n\t" + "aese.8 q1, q14\n\t" + "aesmc.8 q1, q1\n\t" + "vld1.32 {q14}, [lr]!\n\t" + "aese.8 q1, q15\n\t" + "veor.32 q1, q1, q14\n\t" + "vmov r6, r7, d2\n\t" + "vmov r8, r9, d3\n\t" + "vldm.32 r12!, {q2-q9}\n\t" + "vldm.32 r12!, {q10-q12}\n\t" + "mov lr, #0x87\n\t" + "cmp r4, #1\n\t" + "blt L_aes_xts_decrypt_arm32_crypto_256_done_%=\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_256_start_1_%=:\n\t" + "vld1.8 {q0}, [%[in]]!\n\t" + "veor.32 q0, q0, q1\n\t" + "aesd.8 q0, q2\n\t" + "aesimc.8 q0, q0\n\t" + "and r5, lr, r9, asr #31\n\t" + "aesd.8 q0, q3\n\t" + "aesimc.8 q0, q0\n\t" + "lsl r9, r9, #1\n\t" + "aesd.8 q0, q4\n\t" + "aesimc.8 q0, q0\n\t" + "orr r9, r9, r8, lsr #31\n\t" + "aesd.8 q0, q5\n\t" + "aesimc.8 q0, q0\n\t" + "lsl r8, r8, #1\n\t" + "aesd.8 q0, q6\n\t" + "aesimc.8 q0, q0\n\t" + "orr r8, r8, r7, lsr #31\n\t" + "aesd.8 q0, q7\n\t" + "aesimc.8 q0, q0\n\t" + "lsl r7, r7, #1\n\t" + "aesd.8 q0, q8\n\t" + "aesimc.8 q0, q0\n\t" + "orr r7, r7, r6, lsr #31\n\t" + "aesd.8 q0, q9\n\t" + "aesimc.8 q0, q0\n\t" + "eor r6, r5, r6, lsl #1\n\t" + "aesd.8 q0, q10\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q11\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q13}, [r12]!\n\t" + "aesd.8 q0, q12\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aesd.8 q0, q13\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q13}, [r12]!\n\t" + "aesd.8 q0, q14\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]\n\t" + "aesd.8 q0, q13\n\t" + "veor.32 q0, q0, q14\n\t" + "sub r12, r12, #48\n\t" + "veor.32 q0, q0, q1\n\t" + "vmov d2, r6, r7\n\t" + "vmov d3, r8, r9\n\t" + "subs r4, r4, #1\n\t" + "vst1.8 {q0}, [%[out]]!\n\t" + "bne L_aes_xts_decrypt_arm32_crypto_256_start_1_%=\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_256_done_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_xts_decrypt_arm32_crypto_256_partial_done_%=\n\t" + "and r5, lr, r9, asr #31\n\t" + "lsl r9, r9, #1\n\t" + "orr r9, r9, r8, lsr #31\n\t" + "lsl r8, r8, #1\n\t" + "orr r8, r8, r7, lsr #31\n\t" + "lsl r7, r7, #1\n\t" + "orr r7, r7, r6, lsr #31\n\t" + "eor r6, r5, r6, lsl #1\n\t" + "vmov d30, r6, r7\n\t" + "vmov d31, r8, r9\n\t" + "vld1.8 {q0}, [%[in]]!\n\t" + "ldr r4, [sp, #8]\n\t" + "veor.32 q0, q0, q15\n\t" + "aesd.8 q0, q2\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q3\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q4\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q5\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q6\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q7\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q8\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q9\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q10\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q11\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q13}, [r12]!\n\t" + "aesd.8 q0, q12\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aesd.8 q0, q13\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q13}, [r12]!\n\t" + "aesd.8 q0, q14\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]\n\t" + "aesd.8 q0, q13\n\t" + "veor.32 q0, q0, q14\n\t" + "sub r12, r12, #48\n\t" + "veor.32 q0, q0, q15\n\t" + "vst1.32 {q0}, [r4]\n\t" + "add %[out], %[out], #16\n\t" + "mov r5, %[sz]\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_256_start_byte_%=:\n\t" + "ldrb r8, [r4]\n\t" + "ldrb r9, [%[in]], #1\n\t" + "strb r8, [%[out]], #1\n\t" + "strb r9, [r4], #1\n\t" + "subs r5, r5, #1\n\t" + "bgt L_aes_xts_decrypt_arm32_crypto_256_start_byte_%=\n\t" + "sub %[out], %[out], %[sz]\n\t" + "sub r4, r4, %[sz]\n\t" + "sub %[out], %[out], #16\n\t" + "vld1.32 {q0}, [r4]\n\t" + "veor.32 q0, q0, q1\n\t" + "aesd.8 q0, q2\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q3\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q4\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q5\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q6\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q7\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q8\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q9\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q10\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q11\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q13}, [r12]!\n\t" + "aesd.8 q0, q12\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]!\n\t" + "aesd.8 q0, q13\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q13}, [r12]!\n\t" + "aesd.8 q0, q14\n\t" + "aesimc.8 q0, q0\n\t" + "vld1.32 {q14}, [r12]\n\t" + "aesd.8 q0, q13\n\t" + "veor.32 q0, q0, q14\n\t" + "veor.32 q0, q0, q1\n\t" + "vst1.8 {q0}, [%[out]]\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_256_partial_done_%=:\n\t" +#endif /* !NO_AES_256 */ + "b L_aes_xts_decrypt_arm32_crypto_done_%=\n\t" + /* AES_XTS_128 */ + "\n" + "L_aes_xts_decrypt_arm32_crypto_start_128_%=:\n\t" +#ifndef NO_AES_128 + "vldm.32 lr!, {q10-q12}\n\t" + "aese.8 q1, q2\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q3\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q4\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q6\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q7\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q8\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q9\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q10\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q1, q11\n\t" + "veor.32 q1, q1, q12\n\t" + "vmov r6, r7, d2\n\t" + "vmov r8, r9, d3\n\t" + "vldm.32 r12!, {q2-q9}\n\t" + "vldm.32 r12!, {q10-q12}\n\t" + "mov lr, #0x87\n\t" + "cmp r4, #1\n\t" + "blt L_aes_xts_decrypt_arm32_crypto_128_done_%=\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_128_start_1_%=:\n\t" + "vld1.8 {q0}, [%[in]]!\n\t" + "veor.32 q0, q0, q1\n\t" + "aesd.8 q0, q2\n\t" + "aesimc.8 q0, q0\n\t" + "and r5, lr, r9, asr #31\n\t" + "aesd.8 q0, q3\n\t" + "aesimc.8 q0, q0\n\t" + "lsl r9, r9, #1\n\t" + "aesd.8 q0, q4\n\t" + "aesimc.8 q0, q0\n\t" + "orr r9, r9, r8, lsr #31\n\t" + "aesd.8 q0, q5\n\t" + "aesimc.8 q0, q0\n\t" + "lsl r8, r8, #1\n\t" + "aesd.8 q0, q6\n\t" + "aesimc.8 q0, q0\n\t" + "orr r8, r8, r7, lsr #31\n\t" + "aesd.8 q0, q7\n\t" + "aesimc.8 q0, q0\n\t" + "lsl r7, r7, #1\n\t" + "aesd.8 q0, q8\n\t" + "aesimc.8 q0, q0\n\t" + "orr r7, r7, r6, lsr #31\n\t" + "aesd.8 q0, q9\n\t" + "aesimc.8 q0, q0\n\t" + "eor r6, r5, r6, lsl #1\n\t" + "aesd.8 q0, q10\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q11\n\t" + "veor.32 q0, q0, q12\n\t" + "veor.32 q0, q0, q1\n\t" + "vmov d2, r6, r7\n\t" + "vmov d3, r8, r9\n\t" + "subs r4, r4, #1\n\t" + "vst1.8 {q0}, [%[out]]!\n\t" + "bne L_aes_xts_decrypt_arm32_crypto_128_start_1_%=\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_128_done_%=:\n\t" + "cmp %[sz], #0\n\t" + "beq L_aes_xts_decrypt_arm32_crypto_128_partial_done_%=\n\t" + "and r5, lr, r9, asr #31\n\t" + "lsl r9, r9, #1\n\t" + "orr r9, r9, r8, lsr #31\n\t" + "lsl r8, r8, #1\n\t" + "orr r8, r8, r7, lsr #31\n\t" + "lsl r7, r7, #1\n\t" + "orr r7, r7, r6, lsr #31\n\t" + "eor r6, r5, r6, lsl #1\n\t" + "vmov d30, r6, r7\n\t" + "vmov d31, r8, r9\n\t" + "vld1.8 {q0}, [%[in]]!\n\t" + "ldr r4, [sp, #8]\n\t" + "veor.32 q0, q0, q15\n\t" + "aesd.8 q0, q2\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q3\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q4\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q5\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q6\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q7\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q8\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q9\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q10\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q11\n\t" + "veor.32 q0, q0, q12\n\t" + "veor.32 q0, q0, q15\n\t" + "vst1.32 {q0}, [r4]\n\t" + "add %[out], %[out], #16\n\t" + "mov r5, %[sz]\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_128_start_byte_%=:\n\t" + "ldrb r8, [r4]\n\t" + "ldrb r9, [%[in]], #1\n\t" + "strb r8, [%[out]], #1\n\t" + "strb r9, [r4], #1\n\t" + "subs r5, r5, #1\n\t" + "bgt L_aes_xts_decrypt_arm32_crypto_128_start_byte_%=\n\t" + "sub %[out], %[out], %[sz]\n\t" + "sub r4, r4, %[sz]\n\t" + "sub %[out], %[out], #16\n\t" + "vld1.32 {q0}, [r4]\n\t" + "veor.32 q0, q0, q1\n\t" + "aesd.8 q0, q2\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q3\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q4\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q5\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q6\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q7\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q8\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q9\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q10\n\t" + "aesimc.8 q0, q0\n\t" + "aesd.8 q0, q11\n\t" + "veor.32 q0, q0, q12\n\t" + "veor.32 q0, q0, q1\n\t" + "vst1.8 {q0}, [%[out]]\n\t" + "\n" + "L_aes_xts_decrypt_arm32_crypto_128_partial_done_%=:\n\t" +#endif /* !NO_AES_128 */ + "\n" + "L_aes_xts_decrypt_arm32_crypto_done_%=:\n\t" + "pop {%[key], %[key2]}\n\t" + "pop {%[tmp], %[nr], %[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), [i] "+r" (i), + [key] "+r" (key), [key2] "+r" (key2), [tmp] "+r" (tmp), + [nr] "+r" (nr), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "ubfx r7, r10, #0, #8\n\t" -#endif - "eor r6, r6, lr, lsl #16\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl lr, r9, #16\n\t" - "lsr lr, lr, #24\n\t" + : + : [in] "r" (in), [out] "r" (out), [sz] "r" (sz), [i] "r" (i), + [key] "r" (key), [key2] "r" (key2), [tmp] "r" (tmp), [nr] "r" (nr), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r7", "r8", "r9", "q0", "q1", "q2", "q3", "q4", "q5", + "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + ); +} + +#endif /* HAVE_AES_DECRYPT */ +#endif /* WOLFSSL_AES_XTS */ +#ifdef WOLFSSL_AESGCM_SIV +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_GCMSIV_polyval_crypto(unsigned char* s_p, + const unsigned char* h_p, const unsigned char* data_p, + unsigned int blocks_p) #else - "uxtb lr, r9, ror #8\n\t" -#endif +WC_OMIT_FRAME_POINTER void AES_GCMSIV_polyval_crypto(unsigned char* s, + const unsigned char* h, const unsigned char* data, unsigned int blocks) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register unsigned char* s __asm__ ("r0") = (unsigned char*)s_p; + register const unsigned char* h __asm__ ("r1") = (const unsigned char*)h_p; + register const unsigned char* data __asm__ ("r2") = + (const unsigned char*)data_p; + register unsigned int blocks __asm__ ("r3") = (unsigned int)blocks_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r12") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "ubfx lr, r9, #8, #8\n\t" -#endif - "eor r6, r6, r2, lsl #24\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "lsl r2, r8, #8\n\t" - "lsr r2, r2, #24\n\t" + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_GCMSIV_ctr_base_te]}\n\t" + "veor.8 q2, q2, q2\n\t" + "vld1.8 {q1}, [%[h]]\n\t" + "vld1.8 {q0}, [%[s]]\n\t" + "vrev64.8 q0, q0\n\t" + "vext.8 q0, q0, q0, #8\n\t" + "vmull.p64 q7, d2, d2\n\t" + "vmull.p64 q11, d2, d3\n\t" + "vmull.p64 q12, d3, d2\n\t" + "vmull.p64 q8, d3, d3\n\t" + "veor.8 q11, q11, q12\n\t" + "vext.8 q12, q2, q11, #8\n\t" + "vext.8 q11, q11, q2, #8\n\t" + "veor.8 q7, q7, q12\n\t" + "veor.8 q8, q8, q11\n\t" + "vshr.u32 q6, q7, #31\n\t" + "vshr.u32 q9, q8, #31\n\t" + "vshl.i32 q7, q7, #1\n\t" + "vshl.i32 q8, q8, #1\n\t" + "vext.8 q10, q6, q2, #12\n\t" + "vext.8 q9, q2, q9, #12\n\t" + "vext.8 q6, q2, q6, #12\n\t" + "veor.8 q7, q7, q6\n\t" + "veor.8 q8, q8, q9\n\t" + "veor.8 q8, q8, q10\n\t" + "vshl.i32 q6, q7, #31\n\t" + "vshl.i32 q9, q7, #30\n\t" + "vshl.i32 q10, q7, #25\n\t" + "veor.8 q6, q6, q9\n\t" + "veor.8 q6, q6, q10\n\t" + "vext.8 q9, q6, q2, #4\n\t" + "vext.8 q6, q2, q6, #4\n\t" + "veor.8 q7, q7, q6\n\t" + "vshr.u32 q11, q7, #1\n\t" + "vshr.u32 q12, q7, #2\n\t" + "vshr.u32 q13, q7, #7\n\t" + "veor.8 q11, q11, q12\n\t" + "veor.8 q11, q11, q13\n\t" + "veor.8 q11, q11, q9\n\t" + "veor.8 q7, q7, q11\n\t" + "veor.8 q3, q8, q7\n\t" + "vmull.p64 q7, d6, d2\n\t" + "vmull.p64 q11, d6, d3\n\t" + "vmull.p64 q12, d7, d2\n\t" + "vmull.p64 q8, d7, d3\n\t" + "veor.8 q11, q11, q12\n\t" + "vext.8 q12, q2, q11, #8\n\t" + "vext.8 q11, q11, q2, #8\n\t" + "veor.8 q7, q7, q12\n\t" + "veor.8 q8, q8, q11\n\t" + "vshr.u32 q6, q7, #31\n\t" + "vshr.u32 q9, q8, #31\n\t" + "vshl.i32 q7, q7, #1\n\t" + "vshl.i32 q8, q8, #1\n\t" + "vext.8 q10, q6, q2, #12\n\t" + "vext.8 q9, q2, q9, #12\n\t" + "vext.8 q6, q2, q6, #12\n\t" + "veor.8 q7, q7, q6\n\t" + "veor.8 q8, q8, q9\n\t" + "veor.8 q8, q8, q10\n\t" + "vshl.i32 q6, q7, #31\n\t" + "vshl.i32 q9, q7, #30\n\t" + "vshl.i32 q10, q7, #25\n\t" + "veor.8 q6, q6, q9\n\t" + "veor.8 q6, q6, q10\n\t" + "vext.8 q9, q6, q2, #4\n\t" + "vext.8 q6, q2, q6, #4\n\t" + "veor.8 q7, q7, q6\n\t" + "vshr.u32 q11, q7, #1\n\t" + "vshr.u32 q12, q7, #2\n\t" + "vshr.u32 q13, q7, #7\n\t" + "veor.8 q11, q11, q12\n\t" + "veor.8 q11, q11, q13\n\t" + "veor.8 q11, q11, q9\n\t" + "veor.8 q7, q7, q11\n\t" + "veor.8 q4, q8, q7\n\t" + "vmull.p64 q7, d6, d6\n\t" + "vmull.p64 q11, d6, d7\n\t" + "vmull.p64 q12, d7, d6\n\t" + "vmull.p64 q8, d7, d7\n\t" + "veor.8 q11, q11, q12\n\t" + "vext.8 q12, q2, q11, #8\n\t" + "vext.8 q11, q11, q2, #8\n\t" + "veor.8 q7, q7, q12\n\t" + "veor.8 q8, q8, q11\n\t" + "vshr.u32 q6, q7, #31\n\t" + "vshr.u32 q9, q8, #31\n\t" + "vshl.i32 q7, q7, #1\n\t" + "vshl.i32 q8, q8, #1\n\t" + "vext.8 q10, q6, q2, #12\n\t" + "vext.8 q9, q2, q9, #12\n\t" + "vext.8 q6, q2, q6, #12\n\t" + "veor.8 q7, q7, q6\n\t" + "veor.8 q8, q8, q9\n\t" + "veor.8 q8, q8, q10\n\t" + "vshl.i32 q6, q7, #31\n\t" + "vshl.i32 q9, q7, #30\n\t" + "vshl.i32 q10, q7, #25\n\t" + "veor.8 q6, q6, q9\n\t" + "veor.8 q6, q6, q10\n\t" + "vext.8 q9, q6, q2, #4\n\t" + "vext.8 q6, q2, q6, #4\n\t" + "veor.8 q7, q7, q6\n\t" + "vshr.u32 q11, q7, #1\n\t" + "vshr.u32 q12, q7, #2\n\t" + "vshr.u32 q13, q7, #7\n\t" + "veor.8 q11, q11, q12\n\t" + "veor.8 q11, q11, q13\n\t" + "veor.8 q11, q11, q9\n\t" + "veor.8 q7, q7, q11\n\t" + "veor.8 q5, q8, q7\n\t" + "lsr r12, %[blocks], #2\n\t" + "cmp r12, #0\n\t" + "beq L_AES_GCMSIV_polyval_crypto_rem_start_%=\n\t" + "\n" + "L_AES_GCMSIV_polyval_crypto_group_%=:\n\t" + "vld1.8 {q6}, [%[data]]!\n\t" + "veor.8 q6, q6, q0\n\t" + "vmull.p64 q9, d12, d10\n\t" + "vmull.p64 q11, d12, d11\n\t" + "vmull.p64 q12, d13, d10\n\t" + "vmull.p64 q10, d13, d11\n\t" + "veor.8 q11, q11, q12\n\t" + "vext.8 q12, q2, q11, #8\n\t" + "vext.8 q11, q11, q2, #8\n\t" + "veor.8 q9, q9, q12\n\t" + "veor.8 q10, q10, q11\n\t" + "vld1.8 {q6}, [%[data]]!\n\t" + "vmull.p64 q7, d12, d8\n\t" + "vmull.p64 q11, d12, d9\n\t" + "vmull.p64 q12, d13, d8\n\t" + "vmull.p64 q8, d13, d9\n\t" + "veor.8 q11, q11, q12\n\t" + "vext.8 q12, q2, q11, #8\n\t" + "vext.8 q11, q11, q2, #8\n\t" + "veor.8 q7, q7, q12\n\t" + "veor.8 q8, q8, q11\n\t" + "veor.8 q9, q9, q7\n\t" + "veor.8 q10, q10, q8\n\t" + "vld1.8 {q6}, [%[data]]!\n\t" + "vmull.p64 q7, d12, d6\n\t" + "vmull.p64 q11, d12, d7\n\t" + "vmull.p64 q12, d13, d6\n\t" + "vmull.p64 q8, d13, d7\n\t" + "veor.8 q11, q11, q12\n\t" + "vext.8 q12, q2, q11, #8\n\t" + "vext.8 q11, q11, q2, #8\n\t" + "veor.8 q7, q7, q12\n\t" + "veor.8 q8, q8, q11\n\t" + "veor.8 q9, q9, q7\n\t" + "veor.8 q10, q10, q8\n\t" + "vld1.8 {q6}, [%[data]]!\n\t" + "vmull.p64 q7, d12, d2\n\t" + "vmull.p64 q11, d12, d3\n\t" + "vmull.p64 q12, d13, d2\n\t" + "vmull.p64 q8, d13, d3\n\t" + "veor.8 q11, q11, q12\n\t" + "vext.8 q12, q2, q11, #8\n\t" + "vext.8 q11, q11, q2, #8\n\t" + "veor.8 q7, q7, q12\n\t" + "veor.8 q8, q8, q11\n\t" + "veor.8 q9, q9, q7\n\t" + "veor.8 q10, q10, q8\n\t" + "vshr.u32 q6, q9, #31\n\t" + "vshr.u32 q7, q10, #31\n\t" + "vshl.i32 q9, q9, #1\n\t" + "vshl.i32 q10, q10, #1\n\t" + "vext.8 q8, q6, q2, #12\n\t" + "vext.8 q7, q2, q7, #12\n\t" + "vext.8 q6, q2, q6, #12\n\t" + "veor.8 q9, q9, q6\n\t" + "veor.8 q10, q10, q7\n\t" + "veor.8 q10, q10, q8\n\t" + "vshl.i32 q6, q9, #31\n\t" + "vshl.i32 q7, q9, #30\n\t" + "vshl.i32 q8, q9, #25\n\t" + "veor.8 q6, q6, q7\n\t" + "veor.8 q6, q6, q8\n\t" + "vext.8 q7, q6, q2, #4\n\t" + "vext.8 q6, q2, q6, #4\n\t" + "veor.8 q9, q9, q6\n\t" + "vshr.u32 q11, q9, #1\n\t" + "vshr.u32 q12, q9, #2\n\t" + "vshr.u32 q13, q9, #7\n\t" + "veor.8 q11, q11, q12\n\t" + "veor.8 q11, q11, q13\n\t" + "veor.8 q11, q11, q7\n\t" + "veor.8 q9, q9, q11\n\t" + "veor.8 q0, q10, q9\n\t" + "subs r12, r12, #1\n\t" + "bne L_AES_GCMSIV_polyval_crypto_group_%=\n\t" + "\n" + "L_AES_GCMSIV_polyval_crypto_rem_start_%=:\n\t" + "and %[blocks], %[blocks], #3\n\t" + "cmp %[blocks], #0\n\t" + "beq L_AES_GCMSIV_polyval_crypto_done_%=\n\t" + "\n" + "L_AES_GCMSIV_polyval_crypto_rem_%=:\n\t" + "vld1.8 {q6}, [%[data]]!\n\t" + "veor.8 q0, q0, q6\n\t" + "vmull.p64 q9, d0, d2\n\t" + "vmull.p64 q11, d0, d3\n\t" + "vmull.p64 q12, d1, d2\n\t" + "vmull.p64 q10, d1, d3\n\t" + "veor.8 q11, q11, q12\n\t" + "vext.8 q12, q2, q11, #8\n\t" + "vext.8 q11, q11, q2, #8\n\t" + "veor.8 q9, q9, q12\n\t" + "veor.8 q10, q10, q11\n\t" + "vshr.u32 q6, q9, #31\n\t" + "vshr.u32 q7, q10, #31\n\t" + "vshl.i32 q9, q9, #1\n\t" + "vshl.i32 q10, q10, #1\n\t" + "vext.8 q8, q6, q2, #12\n\t" + "vext.8 q7, q2, q7, #12\n\t" + "vext.8 q6, q2, q6, #12\n\t" + "veor.8 q9, q9, q6\n\t" + "veor.8 q10, q10, q7\n\t" + "veor.8 q10, q10, q8\n\t" + "vshl.i32 q6, q9, #31\n\t" + "vshl.i32 q7, q9, #30\n\t" + "vshl.i32 q8, q9, #25\n\t" + "veor.8 q6, q6, q7\n\t" + "veor.8 q6, q6, q8\n\t" + "vext.8 q7, q6, q2, #4\n\t" + "vext.8 q6, q2, q6, #4\n\t" + "veor.8 q9, q9, q6\n\t" + "vshr.u32 q11, q9, #1\n\t" + "vshr.u32 q12, q9, #2\n\t" + "vshr.u32 q13, q9, #7\n\t" + "veor.8 q11, q11, q12\n\t" + "veor.8 q11, q11, q13\n\t" + "veor.8 q11, q11, q7\n\t" + "veor.8 q9, q9, q11\n\t" + "veor.8 q0, q10, q9\n\t" + "subs %[blocks], %[blocks], #1\n\t" + "bne L_AES_GCMSIV_polyval_crypto_rem_%=\n\t" + "\n" + "L_AES_GCMSIV_polyval_crypto_done_%=:\n\t" + "vrev64.8 q0, q0\n\t" + "vext.8 q0, q0, q0, #8\n\t" + "vst1.8 {q0}, [%[s]]\n\t" + "pop {%[L_AES_GCMSIV_ctr_base_te]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [s] "+r" (s), [h] "+r" (h), [data] "+r" (data), + [blocks] "+r" (blocks), + [L_AES_GCMSIV_ctr_base_te] "+r" (L_AES_GCMSIV_ctr_base_te_c) + : #else - "uxtb r2, r8, ror #16\n\t" -#endif + : + : [s] "r" (s), [h] "r" (h), [data] "r" (data), [blocks] "r" (blocks), + [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", + "q9", "q10", "q11", "q12", "q13" + ); +} + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_crypto(const unsigned char* in_p, + unsigned char* out_p, unsigned long length_p, const unsigned char* KS_p, + int nr_p, unsigned char* ctr_p) #else - "ubfx r2, r8, #16, #8\n\t" -#endif - "ldrb r11, [r0, r11, lsl #2]\n\t" - "ldrb r7, [r0, r7, lsl #2]\n\t" - "ldrb lr, [r0, lr, lsl #2]\n\t" - "ldrb r2, [r0, r2, lsl #2]\n\t" - "eor lr, lr, r11, lsl #16\n\t" - "ldm %[KS], {r8, r9, r10, r11}\n\t" - "eor r7, r7, lr, lsl #8\n\t" - "eor r7, r7, r2, lsl #16\n\t" - /* XOR in Key Schedule */ - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" -#endif /* !WOLFSSL_ARMASM_AES_BLOCK_INLINE */ - "pop {r1, %[length], lr}\n\t" - "ldr %[KS], [sp]\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "eor r8, r4, r4, ror #16\n\t" - "eor r9, r5, r5, ror #16\n\t" - "eor r10, r6, r6, ror #16\n\t" - "eor r11, r7, r7, ror #16\n\t" - "bic r8, r8, #0xff0000\n\t" - "bic r9, r9, #0xff0000\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "ror r5, r5, #8\n\t" - "ror r6, r6, #8\n\t" - "ror r7, r7, #8\n\t" - "eor r4, r4, r8, lsr #8\n\t" - "eor r5, r5, r9, lsr #8\n\t" - "eor r6, r6, r10, lsr #8\n\t" - "eor r7, r7, r11, lsr #8\n\t" +WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_crypto(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const unsigned char* in __asm__ ("r0") = + (const unsigned char*)in_p; + register unsigned char* out __asm__ ("r1") = (unsigned char*)out_p; + register unsigned long length __asm__ ("r2") = (unsigned long)length_p; + register const unsigned char* KS __asm__ ("r3") = + (const unsigned char*)KS_p; + register int nr __asm__ ("r12") = (int)nr_p; + register unsigned char* ctr __asm__ ("lr") = (unsigned char*)ctr_p; + register word32* L_AES_GCMSIV_ctr_base_te_c __asm__ ("r4") = + (word32*)L_AES_GCMSIV_ctr_base_te; #else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "ldr r8, [lr]\n\t" - "ldr r9, [lr, #4]\n\t" - "ldr r10, [lr, #8]\n\t" - "ldr r11, [lr, #12]\n\t" - "eor r4, r4, r8\n\t" - "eor r5, r5, r9\n\t" - "eor r6, r6, r10\n\t" - "eor r7, r7, r11\n\t" - "ldr r8, [sp, #4]\n\t" - "str r4, [%[out]]\n\t" - "str r5, [%[out], #4]\n\t" - "str r6, [%[out], #8]\n\t" - "str r7, [%[out], #12]\n\t" - "ldm r8, {r4, r5, r6, r7}\n\t" - "subs %[length], %[length], #16\n\t" - "add lr, lr, #16\n\t" - "add %[out], %[out], #16\n\t" - "bne L_AES_GCMSIV_ctr_base_loop_block_128_%=\n\t" + register word32* L_AES_GCMSIV_ctr_base_te_c = + (word32*)L_AES_GCMSIV_ctr_base_te; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_AES_GCMSIV_ctr_base_te]}\n\t" + "push {%[nr], %[ctr]}\n\t" + "ldr r12, [sp]\n\t" + "ldr lr, [sp, #4]\n\t" + "vld1.8 {q0}, [lr]\n\t" + "lsr r5, %[length], #4\n\t" + "vmov r7, r8, d0\n\t" + "vmov r9, r10, d1\n\t" "\n" - "L_AES_GCMSIV_ctr_base_end_%=:\n\t" - "pop {%[KS], r8}\n\t" -#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) - "eor r10, r4, r4, ror #16\n\t" - "eor r11, r5, r5, ror #16\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r4, r4, #8\n\t" - "ror r5, r5, #8\n\t" - "eor r4, r4, r10, lsr #8\n\t" - "eor r5, r5, r11, lsr #8\n\t" - "eor r10, r6, r6, ror #16\n\t" - "eor r11, r7, r7, ror #16\n\t" - "bic r10, r10, #0xff0000\n\t" - "bic r11, r11, #0xff0000\n\t" - "ror r6, r6, #8\n\t" - "ror r7, r7, #8\n\t" - "eor r6, r6, r10, lsr #8\n\t" - "eor r7, r7, r11, lsr #8\n\t" -#else - "rev r4, r4\n\t" - "rev r5, r5\n\t" - "rev r6, r6\n\t" - "rev r7, r7\n\t" -#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ - "stm r8, {r4, r5, r6, r7}\n\t" + "L_AES_GCMSIV_ctr_crypto_loop2_%=:\n\t" + "cmp r5, #2\n\t" + "blt L_AES_GCMSIV_ctr_crypto_tail_%=\n\t" + "vmov d2, r7, r8\n\t" + "vmov d3, r9, r10\n\t" + "add %[length], r7, #1\n\t" + "vmov d4, %[length], r8\n\t" + "vmov d5, r9, r10\n\t" + "add r7, r7, #2\n\t" + "mov r4, %[KS]\n\t" + "sub r6, r12, #1\n\t" + "\n" + "L_AES_GCMSIV_ctr_crypto_rounds2_%=:\n\t" + "vld1.32 {q5}, [r4]!\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "aese.8 q2, q5\n\t" + "aesmc.8 q2, q2\n\t" + "subs r6, r6, #1\n\t" + "bne L_AES_GCMSIV_ctr_crypto_rounds2_%=\n\t" + "vld1.32 {q5}, [r4]!\n\t" + "aese.8 q1, q5\n\t" + "aese.8 q2, q5\n\t" + "vld1.32 {q5}, [r4]\n\t" + "veor.8 q1, q1, q5\n\t" + "veor.8 q2, q2, q5\n\t" + "vld1.8 {q3-q4}, [%[in]]!\n\t" + "veor.8 q3, q3, q1\n\t" + "veor.8 q4, q4, q2\n\t" + "vst1.8 {q3-q4}, [%[out]]!\n\t" + "sub r5, r5, #2\n\t" + "b L_AES_GCMSIV_ctr_crypto_loop2_%=\n\t" + "\n" + "L_AES_GCMSIV_ctr_crypto_tail_%=:\n\t" + "cmp r5, #0\n\t" + "beq L_AES_GCMSIV_ctr_crypto_done_%=\n\t" + "vmov d2, r7, r8\n\t" + "vmov d3, r9, r10\n\t" + "add r7, r7, #1\n\t" + "mov r4, %[KS]\n\t" + "sub r6, r12, #1\n\t" + "\n" + "L_AES_GCMSIV_ctr_crypto_rounds1_%=:\n\t" + "vld1.32 {q5}, [r4]!\n\t" + "aese.8 q1, q5\n\t" + "aesmc.8 q1, q1\n\t" + "subs r6, r6, #1\n\t" + "bne L_AES_GCMSIV_ctr_crypto_rounds1_%=\n\t" + "vld1.32 {q5}, [r4]!\n\t" + "aese.8 q1, q5\n\t" + "vld1.32 {q5}, [r4]\n\t" + "veor.8 q1, q1, q5\n\t" + "vld1.8 {q3}, [%[in]]!\n\t" + "veor.8 q3, q3, q1\n\t" + "vst1.8 {q3}, [%[out]]!\n\t" + "\n" + "L_AES_GCMSIV_ctr_crypto_done_%=:\n\t" + "vmov d0, r7, r8\n\t" + "vst1.8 {q0}, [lr]\n\t" "pop {%[nr], %[ctr]}\n\t" "pop {%[L_AES_GCMSIV_ctr_base_te]}\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -30548,7 +30668,8 @@ WC_OMIT_FRAME_POINTER void AES_GCMSIV_ctr_base(const unsigned char* in, [nr] "r" (nr), [ctr] "r" (ctr), [L_AES_GCMSIV_ctr_base_te] "r" (L_AES_GCMSIV_ctr_base_te_c) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ - : "memory", "cc", "r5", "r6", "r7", "r8", "r9", "r10", "r11" + : "memory", "cc", "r5", "r6", "r7", "r8", "r9", "r10", "q0", "q1", "q2", + "q3", "q4", "q5" ); } diff --git a/wolfcrypt/src/port/arm/armv8-32-sha256-asm.S b/wolfcrypt/src/port/arm/armv8-32-sha256-asm.S index 6515bd8484..50b3f25483 100644 --- a/wolfcrypt/src/port/arm/armv8-32-sha256-asm.S +++ b/wolfcrypt/src/port/arm/armv8-32-sha256-asm.S @@ -2528,7 +2528,7 @@ L_SHA256_transform_len_blk_end_15: pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} .size Transform_Sha256_Len_base,.-Transform_Sha256_Len_base #else -#ifdef WOLFSSL_ARMASM_NO_HW_CRYPTO +#if !defined(WOLFSSL_ARMASM_NO_NEON_IMPL) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) #ifndef __APPLE__ .text .type L_SHA256_transform_neon_len_k, %object @@ -3588,7 +3588,8 @@ L_SHA256_transform_neon_len_start: vpop {d8-d11} pop {r4, r5, r6, r7, r8, r9, r10, pc} .size Transform_Sha256_Len_neon,.-Transform_Sha256_Len_neon -#else +#endif /* !WOLFSSL_ARMASM_NO_NEON_IMPL || WOLFSSL_ARMASM_NO_HW_CRYPTO */ +#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #ifndef __APPLE__ .text .type L_SHA256_trans_crypto_len_k, %object @@ -3621,6 +3622,7 @@ L_SHA256_trans_crypto_len_k: .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 .text .align 4 + .fpu crypto-neon-fp-armv8 .globl Transform_Sha256_Len_crypto .type Transform_Sha256_Len_crypto, %function Transform_Sha256_Len_crypto: diff --git a/wolfcrypt/src/port/arm/armv8-32-sha256-asm_c.c b/wolfcrypt/src/port/arm/armv8-32-sha256-asm_c.c index 18c3b5c4b3..86717b71b8 100644 --- a/wolfcrypt/src/port/arm/armv8-32-sha256-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-32-sha256-asm_c.c @@ -2583,7 +2583,8 @@ WC_OMIT_FRAME_POINTER void Transform_Sha256_Len_base(wc_Sha256* sha256, } #else -#ifdef WOLFSSL_ARMASM_NO_HW_CRYPTO +#if !defined(WOLFSSL_ARMASM_NO_NEON_IMPL) || \ + defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) XALIGNED(8) static const word32 L_SHA256_transform_neon_len_k[] = { 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, @@ -3660,7 +3661,8 @@ WC_OMIT_FRAME_POINTER void Transform_Sha256_Len_neon(wc_Sha256* sha256, ); } -#else +#endif /* !WOLFSSL_ARMASM_NO_NEON_IMPL || WOLFSSL_ARMASM_NO_HW_CRYPTO */ +#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO XALIGNED(8) static const word32 L_SHA256_trans_crypto_len_k[] = { 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, diff --git a/wolfcrypt/src/sha256.c b/wolfcrypt/src/sha256.c index 4c3d4fd2eb..637f58c666 100644 --- a/wolfcrypt/src/sha256.c +++ b/wolfcrypt/src/sha256.c @@ -1342,6 +1342,48 @@ int wc_InitSha256_ex(wc_Sha256* sha256, void* heap, int devId) #elif defined(WOLFSSL_ARMASM) && !defined(WOLF_CRYPTO_CB_ONLY_SHA256) +/* On 32-bit Arm a NEON build compiles in both the NEON and (unless disabled) + * the Armv8 crypto-extension block transforms, so the best supported one is + * chosen at run time - mirroring the AArch64 path. Thumb-2, no-NEON, no-crypto + * (NO_HW_CRYPTO) and crypto-only (NO_NEON_IMPL) builds compile a single variant + * and call it directly. */ +#if !defined(WOLFSSL_ARMASM_THUMB2) && !defined(WOLFSSL_ARMASM_NO_NEON) && \ + !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) && \ + !defined(WOLFSSL_ARMASM_NO_NEON_IMPL) + #define SHA256_ARM32_DISPATCH +#endif + +#ifdef SHA256_ARM32_DISPATCH + +static int sha256_transform_check = 0; +static cpuid_flags_atomic_t sha256_cpuid_flags = WC_CPUID_ATOMIC_INITIALIZER; + +/* Initialize to the NEON transform: it is valid on any Armv8-32/NEON CPU, so + * the pointer is safe to use even if read before Sha256_SetTransform() runs. */ +static void (*Transform_Sha256_Len_p)(wc_Sha256* sha256, const byte* data, + word32 len) = Transform_Sha256_Len_neon; + +/* Select the crypto-extension transform when the CPU implements FEAT_SHA256, + * otherwise stay on NEON. */ +static void Sha256_SetTransform(void) +{ + if (sha256_transform_check) + return; + + cpuid_get_flags_atomic(&sha256_cpuid_flags); + + if (IS_ARM32_SHA256(sha256_cpuid_flags)) + Transform_Sha256_Len_p = Transform_Sha256_Len_crypto; + else + Transform_Sha256_Len_p = Transform_Sha256_Len_neon; + + sha256_transform_check = 1; +} + +#else +#define Sha256_SetTransform() WC_DO_NOTHING +#endif /* SHA256_ARM32_DISPATCH */ + int wc_InitSha256_ex(wc_Sha256* sha256, void* heap, int devId) { int ret = 0; @@ -1352,6 +1394,8 @@ int wc_InitSha256_ex(wc_Sha256* sha256, void* heap, int devId) if (ret != 0) return ret; + Sha256_SetTransform(); + sha256->heap = heap; #ifdef WOLF_CRYPTO_CB sha256->devId = devId; @@ -1373,8 +1417,10 @@ static WC_INLINE int Transform_Sha256(wc_Sha256* sha256, const byte* data) Transform_Sha256_Len_base(sha256, data, WC_SHA256_BLOCK_SIZE); #elif defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) Transform_Sha256_Len_neon(sha256, data, WC_SHA256_BLOCK_SIZE); -#else +#elif defined(WOLFSSL_ARMASM_NO_NEON_IMPL) Transform_Sha256_Len_crypto(sha256, data, WC_SHA256_BLOCK_SIZE); +#else + (*Transform_Sha256_Len_p)(sha256, data, WC_SHA256_BLOCK_SIZE); #endif return 0; } @@ -1386,8 +1432,10 @@ static WC_INLINE int Transform_Sha256_Len(wc_Sha256* sha256, const byte* data, Transform_Sha256_Len_base(sha256, data, len); #elif defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) Transform_Sha256_Len_neon(sha256, data, len); -#else +#elif defined(WOLFSSL_ARMASM_NO_NEON_IMPL) Transform_Sha256_Len_crypto(sha256, data, len); +#else + (*Transform_Sha256_Len_p)(sha256, data, len); #endif return 0; } diff --git a/wolfssl/wolfcrypt/aes.h b/wolfssl/wolfcrypt/aes.h index 183aad072f..48462f9272 100644 --- a/wolfssl/wolfcrypt/aes.h +++ b/wolfssl/wolfcrypt/aes.h @@ -331,14 +331,21 @@ struct Aes { #define WC_FLAG_DONT_USE_VECTOR_OPS 2 #endif #endif /* WOLFSSL_AESNI */ -#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ - !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +/* Run-time implementation-selection flags. Present when a hardware-crypto and + * a fallback implementation are both compiled in and chosen at run time: always + * on AArch64, and on 32-bit Arm unless the base fallback is dropped + * (WOLFSSL_ARMASM_NO_BASE_IMPL) or crypto is disabled. */ +#if defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) && \ + (defined(__aarch64__) || !defined(WOLFSSL_ARMASM_NO_BASE_IMPL)) byte use_aes_hw_crypto; #ifdef HAVE_AESGCM byte use_pmull_hw_crypto; +#ifdef __aarch64__ byte use_sha3_hw_crypto; #endif -#endif /* __aarch64__ && WOLFSSL_ARMASM && !WOLFSSL_ARMASM_NO_HW_CRYPTO */ +#endif +#endif /* WOLFSSL_ARMASM && !WOLFSSL_ARMASM_NO_HW_CRYPTO && + * (__aarch64__ || !WOLFSSL_ARMASM_NO_BASE_IMPL) */ #if defined(WOLF_CRYPTO_CB) int devId; void* devCtx; /* Opaque handle for CryptoCB device */ @@ -937,7 +944,11 @@ WOLFSSL_API int wc_AesCtsDecryptFinal(Aes* aes, byte* out, word32* outSz); #endif #if defined(WOLFSSL_ARMASM) -#if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) +/* Base (table) AES is available on AArch64, in a no-crypto build, and in a + * 32-bit crypto build that keeps the base fallback for run-time selection + * (i.e. unless WOLFSSL_ARMASM_NO_BASE_IMPL drops it). */ +#if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ + !defined(WOLFSSL_ARMASM_NO_BASE_IMPL) WOLFSSL_LOCAL void AES_set_encrypt_key(const unsigned char* key, word32 len, unsigned char* ks); WOLFSSL_LOCAL void AES_invert_key(unsigned char* ks, word32 rounds); diff --git a/wolfssl/wolfcrypt/cpuid.h b/wolfssl/wolfcrypt/cpuid.h index faf96c6159..12375fd546 100644 --- a/wolfssl/wolfcrypt/cpuid.h +++ b/wolfssl/wolfcrypt/cpuid.h @@ -43,6 +43,15 @@ #define HAVE_CPUID #define HAVE_CPUID_AARCH64 #endif +/* 32-bit Arm (AArch32 / Thumb-2). __arm__ is defined for both the A32 and + * Thumb instruction sets but not for AArch64, so it cleanly distinguishes the + * 32-bit build from the AArch64 one handled above. */ +#if (defined(WOLFSSL_ARM32_BUILD) || (defined(__arm__) && \ + defined(WOLFSSL_ARMASM))) && !defined(__aarch64__) && \ + !defined(WOLFSSL_NO_ASM) + #define HAVE_CPUID + #define HAVE_CPUID_ARM32 +#endif #if defined(WOLFSSL_PPC64_ASM) && !defined(WOLFSSL_NO_ASM) #define HAVE_CPUID #define HAVE_CPUID_PPC64 @@ -121,6 +130,25 @@ typedef word32 cpuid_flags_t; #define IS_AARCH64_SVE(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_SVE) #define IS_AARCH64_SME(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_SME) +#elif defined(HAVE_CPUID_ARM32) + + /* Only the features that select between the available wolfCrypt AArch32 + * assembly implementations are tracked: + * - AES / PMULL: Armv8 crypto-extension AES and AES-GCM (armv8-32-aes). + * - SHA256: Armv8 crypto-extension SHA-256 (armv8-32-sha256). + * - ASIMD (NEON): armv8-32 SHA-512, SHA-3, ChaCha and Poly1305 asm. + * AArch32 has no crypto-extension SHA-512/SHA-3 instructions, so those + * digests are accelerated with NEON rather than a crypto path. */ + #define CPUID_AES 0x0001 /* AES enc/dec */ + #define CPUID_PMULL 0x0002 /* Carryless multiply - VMULL.P64 */ + #define CPUID_SHA256 0x0004 /* SHA-256 digest */ + #define CPUID_ASIMD 0x0008 /* Advanced SIMD - NEON */ + + #define IS_ARM32_AES(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_AES) + #define IS_ARM32_PMULL(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_PMULL) + #define IS_ARM32_SHA256(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_SHA256) + #define IS_ARM32_ASIMD(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_ASIMD) + #elif defined(HAVE_CPUID_PPC64) #define CPUID_ALTIVEC 0x0001 /* VMX / AltiVec */