diff --git a/crypto/aes/asm/aes-riscv64-zvbb-zvkg-zvkned.pl b/crypto/aes/asm/aes-riscv64-zvbb-zvkg-zvkned.pl index 147d61fdc396e..c6af9fc39cc22 100644 --- a/crypto/aes/asm/aes-riscv64-zvbb-zvkg-zvkned.pl +++ b/crypto/aes/asm/aes-riscv64-zvbb-zvkg-zvkned.pl @@ -35,7 +35,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - RV64I -# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Vector ('V') with 2048 > VLEN >= 128 # - RISC-V Vector Bit-manipulation extension ('Zvbb') # - RISC-V Vector GCM/GMAC extension ('Zvkg') # - RISC-V Vector AES block cipher extension ('Zvkned') @@ -395,6 +395,53 @@ sub handle_xts_dec_last_block { return $code; } +# prepare xts dec second to last block's input(v24) and iv(v29) and +# last block's and iv(v28) +sub handle_xts_dec_last_block_x2 { + my $code=<<___; + bnez $TAIL_LENGTH, 1f + ret +1: + # load second to last block's ciphertext + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V24, $INPUT]} + addi $INPUT, $INPUT, 16 + + # setup `x` multiplier with byte-reversed order + # 0b00000010 => 0b01000000 (0x40) + li $T0, 0x40 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vmv_v_i $V20, 0]} + @{[vsetivli "zero", 1, "e8", "m1", "tu", "ma"]} + @{[vmv_v_x $V20, $T0]} + + beqz $LENGTH, 1f + # slidedown third to last block + addi $VL, $VL, -4 + @{[vsetivli "zero", 4, "e32", "m2", "ta", "ma"]} + # multiplier + @{[vslidedown_vx $V16, $V16, $VL]} + + # compute IV for last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vgmul_vv $V16, $V20]} + @{[vbrev8_v $V28, $V16]} + + # compute IV for second to last block + @{[vgmul_vv $V16, $V20]} + @{[vbrev8_v $V29, $V16]} + j 2f +1: + # compute IV for second to last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vgmul_vv $V16, $V20]} + @{[vbrev8_v $V29, $V16]} +2: +___ + + return $code; +} + # Load all 11 round keys to v1-v11 registers. sub aes_128_load_key { my $code=<<___; @@ -541,6 +588,46 @@ sub aes_128_dec { return $code; } +# aes-128 dec with round keys v1-v11 +sub aes_128_dec_x2 { + my $code=<<___; + @{[vaesz_vs $V24, $V11]} + @{[vaesz_vs $V26, $V11]} + + @{[vaesdm_vs $V24, $V10]} + @{[vaesdm_vs $V26, $V10]} + + @{[vaesdm_vs $V24, $V9]} + @{[vaesdm_vs $V26, $V9]} + + @{[vaesdm_vs $V24, $V8]} + @{[vaesdm_vs $V26, $V8]} + + @{[vaesdm_vs $V24, $V7]} + @{[vaesdm_vs $V26, $V7]} + + @{[vaesdm_vs $V24, $V6]} + @{[vaesdm_vs $V26, $V6]} + + @{[vaesdm_vs $V24, $V5]} + @{[vaesdm_vs $V26, $V5]} + + @{[vaesdm_vs $V24, $V4]} + @{[vaesdm_vs $V26, $V4]} + + @{[vaesdm_vs $V24, $V3]} + @{[vaesdm_vs $V26, $V3]} + + @{[vaesdm_vs $V24, $V2]} + @{[vaesdm_vs $V26, $V2]} + + @{[vaesdf_vs $V24, $V1]} + @{[vaesdf_vs $V26, $V1]} +___ + + return $code; +} + # aes-256 enc with round keys v1-v15 sub aes_256_enc { my $code=<<___; @@ -813,16 +900,91 @@ sub aes_256_dec { $code .= <<___; .p2align 3 aes_xts_dec_128: - @{[init_first_round]} + @{[init_first_round_x2]} @{[aes_128_load_key]} + csrr $T0, vlenb + blt $LEN32, $T0, .Lsmall + @{[vsetvli $VL, $LEN32, "e32", "m2", "ta", "ma"]} + slli $T1, $VL, 2 + add $INPUT, $INPUT, $T1 + @{[vle32_v $V26, $INPUT]} + add $INPUT, $INPUT, $T1 + @{[vmv_v_v $V18, $V16]} + @{[vgmul_vv $V18, $V20]} + @{[vbrev8_v $V30, $V18]} + + j 4f +.Ldec_4blocks_loop_128: + blt $LEN32, $T0, .Ldec_2blocks_128 + @{[vle32_v $V24, $INPUT]} + add $INPUT, $INPUT, $T1 + @{[vle32_v $V26, $INPUT]} + add $INPUT, $INPUT, $T1 + # update iv + @{[vgmul_vv $V16, $V12]} + @{[vgmul_vv $V18, $V12]} + # reverse the iv's bits order back + @{[vbrev8_v $V28, $V16]} + @{[vbrev8_v $V30, $V18]} +4: + @{[vxor_vv $V24, $V24, $V28]} + @{[vxor_vv $V26, $V26, $V30]} + @{[aes_128_dec_x2]} + @{[vxor_vv $V24, $V24, $V28]} + @{[vxor_vv $V26, $V26, $V30]} + + @{[vse32_v $V24, $OUTPUT]} + add $OUTPUT, $OUTPUT, $T1 + @{[vse32_v $V26, $OUTPUT]} + add $OUTPUT, $OUTPUT, $T1 + + sub $LEN32, $LEN32, $T0 + bnez $LEN32, .Ldec_4blocks_loop_128 + + bnez $TAIL_LENGTH, 3f + ret +3: + @{[vmv_v_v $V16, $V18]} + @{[handle_xts_dec_last_block_x2]} + + ## xts second to last block + @{[vxor_vv $V24, $V24, $V29]} + @{[aes_128_dec]} + @{[vxor_vv $V24, $V24, $V29]} + @{[vmv_v_v $V25, $V24]} + + # load last block ciphertext + @{[vsetvli "zero", $TAIL_LENGTH, "e8", "m1", "tu", "ma"]} + @{[vle8_v $V24, $INPUT]} + + # store second to last block plaintext + addi $T0, $OUTPUT, 16 + @{[vse8_v $V25, $T0]} + + ## xts last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vxor_vv $V24, $V24, $V28]} + @{[aes_128_dec]} + @{[vxor_vv $V24, $V24, $V28]} + + # store last block plaintext + @{[vse32_v $V24, $OUTPUT]} + + ret + +.Ldec_2blocks_128: + @{[vmv_v_v $V16, $V18]} + j .Ldec_blocks_128 + +.Lsmall: beqz $LEN32, 2f - @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + @{[vsetvli $VL, $LEN32, "e32", "m2", "ta", "ma"]} j 1f .Ldec_blocks_128: - @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + @{[vsetvli $VL, $LEN32, "e32", "m2", "ta", "ma"]} # load ciphertext into v24 @{[vle32_v $V24, $INPUT]} # update iv @@ -844,7 +1006,7 @@ sub aes_256_dec { bnez $LEN32, .Ldec_blocks_128 2: - @{[handle_xts_dec_last_block]} + @{[handle_xts_dec_last_block_x2]} ## xts second to last block @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} diff --git a/providers/implementations/ciphers/cipher_aes_hw_rv64i.c b/providers/implementations/ciphers/cipher_aes_hw_rv64i.c index fa1e3f2b2adff..79271ffcc0ae4 100644 --- a/providers/implementations/ciphers/cipher_aes_hw_rv64i.c +++ b/providers/implementations/ciphers/cipher_aes_hw_rv64i.c @@ -296,7 +296,7 @@ const PROV_CCM_HW *ossl_prov_aes_hw_ccm_rv64i(void) static int cipher_hw_aes_xts_rv64i_initkey(PROV_CIPHER_CTX *ctx, const unsigned char *key, size_t keylen) { - if (RISCV_HAS_ZVBB() && RISCV_HAS_ZVKG() && RISCV_HAS_ZVKNED() && riscv_vlen() >= 128) { + if (RISCV_HAS_ZVBB() && RISCV_HAS_ZVKG() && RISCV_HAS_ZVKNED() && riscv_vlen() >= 128 && riscv_vlen() < 2048) { /* Zvkned only supports 128 and 256 bit keys. */ if (zvkned_key_schedule_supported(keylen / 2)) { return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,