diff --git a/.github/workflows/async-examples.yml b/.github/workflows/async-examples.yml index caca4b6c32b..74ae7f01e83 100644 --- a/.github/workflows/async-examples.yml +++ b/.github/workflows/async-examples.yml @@ -188,3 +188,25 @@ jobs: cat "$f" fi done + + # Crypto-callback record-cipher poll completion (WOLF_CRYPTO_CB_ASYNC_POLL): + # build the unit test with the feature macro and run the async test group. + cryptocb_async_poll: + if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + name: Crypto-cb async poll completion + steps: + - uses: actions/checkout@v5 + name: Checkout wolfSSL + + - name: Build with WOLF_CRYPTO_CB_ASYNC_POLL + run: | + ./autogen.sh + ./configure --enable-asynccrypt --enable-cryptocb --enable-tls13 \ + --enable-des3 --enable-aesccm --enable-aesctr \ + CPPFLAGS=-DWOLF_CRYPTO_CB_ASYNC_POLL + make + + - name: Run async unit tests + run: ./tests/unit.test --group async diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index b506fe1987e..3abcf2beba1 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -1066,6 +1066,7 @@ WOLFSSL_XILINX_PATCH WOLFSSL_XIL_MSG_NO_SLEEP WOLFSSL_ZEPHYR WOLF_ALLOW_BUILTIN +WOLF_CRYPTO_CB_ASYNC_POLL WOLF_CRYPTO_CB_CMD WOLF_CRYPTO_DEV WOLF_NO_TRAILING_ENUM_COMMAS diff --git a/configure.ac b/configure.ac index 4c231e5e400..1edf23285ac 100644 --- a/configure.ac +++ b/configure.ac @@ -11290,6 +11290,23 @@ then fi fi +# Crypto callbacks with async crypt may not work for TLS unless +# WOLF_CRYPTO_CB_ASYNC_POLL is defined. Warn once here and silence the +# source-level #warning. +if test "$ENABLED_ASYNCCRYPT" = "yes" && test "x$ENABLED_CRYPTOCB" != "xno" && + test "x$ENABLED_ASYNCCRYPT_SW" != "xyes" && + test "x$ENABLED_CAVIUM" != "xyes" && test "x$ENABLED_INTEL_QA" != "xyes" +then + case "$CPPFLAGS $CFLAGS $AM_CFLAGS" in + *WOLF_CRYPTO_CB_ASYNC_POLL*) + ;; + *) + AC_MSG_WARN([crypto callbacks with async crypt may not work for TLS. Define WOLF_CRYPTO_CB_ASYNC_POLL to enable it.]) + AM_CFLAGS="$AM_CFLAGS -DWOLF_CRYPTO_CB_ASYNC_NO_WARN" + ;; + esac +fi + # check for async if using Intel QuckAssist or Cavium if test "x$ENABLED_INTEL_QA" = "xyes" || test "x$ENABLED_CAVIUM" = "xyes" ; then if test "x$ENABLED_ASYNCCRYPT" = "xno" ; then diff --git a/examples/async/user_settings.h b/examples/async/user_settings.h index 33ab0b0f3ac..811b2239100 100644 --- a/examples/async/user_settings.h +++ b/examples/async/user_settings.h @@ -28,6 +28,11 @@ #define NO_FILESYSTEM #define WOLFSSL_IGNORE_FILE_WARN +#ifdef WOLF_CRYPTO_CB + /* PK-only offload: silence pending-bulk-cipher warning. */ + #define WOLF_CRYPTO_CB_ASYNC_NO_WARN +#endif + #define HAVE_ECC #define WC_ECC_NONBLOCK #define WC_ECC_NONBLOCK_ONLY diff --git a/src/internal.c b/src/internal.c index 0ea1b04f2b7..d4deba17f03 100644 --- a/src/internal.c +++ b/src/internal.c @@ -21583,7 +21583,16 @@ static WC_INLINE int Encrypt(WOLFSSL* ssl, byte* out, const byte* input, #ifdef WOLFSSL_ASYNC_CRYPT /* If pending, then leave and return will resume below */ if (ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { + #if defined(WOLF_CRYPTO_CB) && \ + !defined(WOLF_CRYPTO_CB_ASYNC_POLL) && \ + !defined(WOLFSSL_ASYNC_CRYPT_SW) && \ + !defined(HAVE_INTEL_QA) && !defined(HAVE_CAVIUM) + /* No completion path for a pending bulk cipher op. */ + WOLFSSL_ERROR_VERBOSE(ASYNC_OP_E); + return ASYNC_OP_E; + #else return ret; + #endif } #endif } @@ -22085,7 +22094,16 @@ static int DecryptTls(WOLFSSL* ssl, byte* plain, const byte* input, word16 sz) #ifdef WOLFSSL_ASYNC_CRYPT /* If pending, leave and return below */ if (ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { + #if defined(WOLF_CRYPTO_CB) && \ + !defined(WOLF_CRYPTO_CB_ASYNC_POLL) && \ + !defined(WOLFSSL_ASYNC_CRYPT_SW) && \ + !defined(HAVE_INTEL_QA) && !defined(HAVE_CAVIUM) + /* No completion path for a pending bulk cipher op. */ + WOLFSSL_ERROR_VERBOSE(ASYNC_OP_E); + return ASYNC_OP_E; + #else return ret; + #endif } #endif } @@ -27850,10 +27868,8 @@ int SendData(WOLFSSL* ssl, const void* data, size_t sz) #endif } if (sendSz < 0) { - #ifdef WOLFSSL_ASYNC_CRYPT - if (sendSz == WC_NO_ERR_TRACE(WC_PENDING_E)) - ssl->error = sendSz; - #endif + /* Preserve the reason for wolfSSL_get_error(). */ + ssl->error = sendSz; return BUILD_MSG_ERROR; } @@ -43928,7 +43944,13 @@ int wolfSSL_AsyncPop(WOLFSSL* ssl, byte* state) #if (defined(WOLF_CRYPTO_CB) || defined(HAVE_PK_CALLBACKS)) && \ !defined(WOLFSSL_ASYNC_CRYPT_SW) && !defined(HAVE_INTEL_QA) && \ !defined(HAVE_CAVIUM) - else if (ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { + else if (ret == WC_NO_ERR_TRACE(WC_PENDING_E) + #if defined(WOLF_CRYPTO_CB) && defined(WOLF_CRYPTO_CB_ASYNC_POLL) + /* Poll-completed ops stay queued until the poll fills the + * output buffer. */ + && asyncDev->cryptocb.devId == INVALID_DEVID + #endif + ) { /* Allow the underlying crypto API to be called again to trigger the * crypto or PK callback. The actual callback must be called, since * the completion is not detected in the poll like Intel QAT or diff --git a/src/tls13.c b/src/tls13.c index b8c7dbbc912..2fa91dda398 100644 --- a/src/tls13.c +++ b/src/tls13.c @@ -2818,6 +2818,14 @@ static int EncryptTls13(WOLFSSL* ssl, byte* output, const byte* input, #ifdef WOLFSSL_ASYNC_CRYPT if (ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { + #if defined(WOLF_CRYPTO_CB) && \ + !defined(WOLF_CRYPTO_CB_ASYNC_POLL) && \ + !defined(WOLFSSL_ASYNC_CRYPT_SW) && \ + !defined(HAVE_INTEL_QA) && !defined(HAVE_CAVIUM) + /* No completion path for a pending bulk cipher op. */ + WOLFSSL_ERROR_VERBOSE(ASYNC_OP_E); + return ASYNC_OP_E; + #else /* if async is not okay, then block */ if (!asyncOkay) { ret = wc_AsyncWait(ret, asyncDev, event_flags); @@ -2826,6 +2834,7 @@ static int EncryptTls13(WOLFSSL* ssl, byte* output, const byte* input, /* If pending, then leave and return will resume below */ return wolfSSL_AsyncPush(ssl, asyncDev); } + #endif } #endif } @@ -3217,7 +3226,16 @@ int DecryptTls13(WOLFSSL* ssl, byte* output, const byte* input, word16 sz, #ifdef WOLFSSL_ASYNC_CRYPT /* If pending, leave now */ if (ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { + #if defined(WOLF_CRYPTO_CB) && \ + !defined(WOLF_CRYPTO_CB_ASYNC_POLL) && \ + !defined(WOLFSSL_ASYNC_CRYPT_SW) && \ + !defined(HAVE_INTEL_QA) && !defined(HAVE_CAVIUM) + /* No completion path for a pending bulk cipher op. */ + WOLFSSL_ERROR_VERBOSE(ASYNC_OP_E); + return ASYNC_OP_E; + #else return ret; + #endif } #endif } diff --git a/tests/api.c b/tests/api.c index f2f88a8be9a..9905d957c63 100644 --- a/tests/api.c +++ b/tests/api.c @@ -214,6 +214,7 @@ #include #include #include +#include #include #include #include @@ -37042,6 +37043,8 @@ TEST_CASE testCases[] = { #endif /* Cipher */ + /* Crypto callback async poll completion */ + TEST_ASYNC_DECLS, /* Triple-DES */ TEST_DES3_DECLS, /* Chacha20 */ diff --git a/tests/api/include.am b/tests/api/include.am index 04285d5279d..e4ccf12abb3 100644 --- a/tests/api/include.am +++ b/tests/api/include.am @@ -22,6 +22,7 @@ tests_unit_test_SOURCES += tests/api/test_kdf.c # SHE tests_unit_test_SOURCES += tests/api/test_she.c # Cipher +tests_unit_test_SOURCES += tests/api/test_async.c tests_unit_test_SOURCES += tests/api/test_des3.c tests_unit_test_SOURCES += tests/api/test_chacha.c tests_unit_test_SOURCES += tests/api/test_poly1305.c @@ -145,6 +146,7 @@ EXTRA_DIST += tests/api/test_hmac.h EXTRA_DIST += tests/api/test_cmac.h EXTRA_DIST += tests/api/test_kdf.h EXTRA_DIST += tests/api/test_she.h +EXTRA_DIST += tests/api/test_async.h EXTRA_DIST += tests/api/test_des3.h EXTRA_DIST += tests/api/test_chacha.h EXTRA_DIST += tests/api/test_poly1305.h diff --git a/tests/api/test_async.c b/tests/api/test_async.c new file mode 100644 index 00000000000..767ee72d314 --- /dev/null +++ b/tests/api/test_async.c @@ -0,0 +1,1038 @@ +/* test_async.c + * + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#include + +#ifdef NO_INLINE + #include +#else + #define WOLFSSL_MISC_INCLUDED + #include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Crypto callback async "poll to fill output" completion + * (WOLF_CRYPTO_CB_ASYNC_POLL). + * + * A crypto callback that returns WC_PENDING_E for a TLS record cipher is + * completed at poll time: wolfSSL keeps the async event queued and re-enters + * the callback with WC_ALGO_TYPE_ASYNC_POLL so it can finish the job and fill + * the output buffer. These tests exercise the completion mechanism directly + * with a fake HSM that offloads the record ciphers (AES-GCM, AES-CBC, + * AES-CCM, 3DES-CBC) and declines everything else. + * + * The tests only run when the feature is compiled in; otherwise they skip. + */ + +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) + +#define TEST_ASYNC_HSM_DEVID 0x00005150 +#define TEST_ASYNC_HSM_DEVID2 0x00005151 /* second device, per-peer TLS test */ +#define TEST_ASYNC_MAX_POLLS 16 + +/* Offloading tests run at pend depths 0..TEST_ASYNC_MAX_PEND. Depth N returns + * WC_PENDING_E on submit and N more times at poll before completing (N+1 polls + * total). Must stay below TEST_ASYNC_MAX_POLLS. */ +#define TEST_ASYNC_MAX_PEND 5 + +/* One in-flight offloaded job. */ +typedef struct TestHsmJob { + int active; + int type; /* wc_CipherType */ + int dec; /* 0 = encrypt, 1 = decrypt */ + int pendRemaining; /* polls still returning WC_PENDING_E */ + void* obj; /* Aes* or Des3* */ + byte* out; + const byte* in; + word32 sz; + const byte* iv; + word32 ivSz; + byte* authTag; /* encrypt: output tag */ + const byte* authTagDec; /* decrypt: input tag to verify */ + word32 authTagSz; + const byte* authIn; + word32 authInSz; +} TestHsmJob; + +typedef struct TestHsm { + TestHsmJob job; + int pendPolls; /* polls to stay in-flight before completing */ + int offloadDec; /* also offload decrypt (single-op tests only) */ + int invocations; /* callback entered at all (any algo/type) */ + int submits; /* offloaded (pended) op count */ + int decSubmits; /* offloaded decrypt op count */ + int polls; +} TestHsm; + +/* Run the software cipher into the saved output, with routing disabled so the + * completion does not re-enter this callback. */ +static int test_hsm_complete(TestHsmJob* j) +{ + int ret = WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE); + + switch (j->type) { +#ifdef HAVE_AESGCM + case WC_CIPHER_AES_GCM: { + Aes* aes = (Aes*)j->obj; + int save = aes->devId; + aes->devId = INVALID_DEVID; + if (j->dec) + ret = wc_AesGcmDecrypt(aes, j->out, j->in, j->sz, j->iv, j->ivSz, + j->authTagDec, j->authTagSz, j->authIn, + j->authInSz); + else + ret = wc_AesGcmEncrypt(aes, j->out, j->in, j->sz, j->iv, j->ivSz, + j->authTag, j->authTagSz, j->authIn, + j->authInSz); + aes->devId = save; + break; + } +#endif +#ifdef HAVE_AESCCM + case WC_CIPHER_AES_CCM: { + Aes* aes = (Aes*)j->obj; + int save = aes->devId; + aes->devId = INVALID_DEVID; + if (j->dec) + ret = wc_AesCcmDecrypt(aes, j->out, j->in, j->sz, j->iv, j->ivSz, + j->authTagDec, j->authTagSz, j->authIn, + j->authInSz); + else + ret = wc_AesCcmEncrypt(aes, j->out, j->in, j->sz, j->iv, j->ivSz, + j->authTag, j->authTagSz, j->authIn, + j->authInSz); + aes->devId = save; + break; + } +#endif +#ifdef HAVE_AES_CBC + case WC_CIPHER_AES_CBC: { + Aes* aes = (Aes*)j->obj; + int save = aes->devId; + aes->devId = INVALID_DEVID; + if (j->dec) + ret = wc_AesCbcDecrypt(aes, j->out, j->in, j->sz); + else + ret = wc_AesCbcEncrypt(aes, j->out, j->in, j->sz); + aes->devId = save; + break; + } +#endif +#ifndef NO_DES3 + case WC_CIPHER_DES3: { + Des3* des = (Des3*)j->obj; + int save = des->devId; + des->devId = INVALID_DEVID; + if (j->dec) + ret = wc_Des3_CbcDecrypt(des, j->out, j->in, j->sz); + else + ret = wc_Des3_CbcEncrypt(des, j->out, j->in, j->sz); + des->devId = save; + break; + } +#endif + default: + break; + } + return ret; +} + +/* Fake HSM: pend and poll-complete the record ciphers, decline everything + * else so it falls back to software. */ +static int test_hsm_cb(int devId, wc_CryptoInfo* info, void* ctx) +{ + TestHsm* hsm = (TestHsm*)ctx; + (void)devId; + + if (info == NULL) + return WC_NO_ERR_TRACE(BAD_FUNC_ARG); + + hsm->invocations++; + + if (info->algo_type == WC_ALGO_TYPE_CIPHER) { + TestHsmJob* j = &hsm->job; + + /* Offload decrypt only when asked. The single job slot cannot hold a + * client decrypt and a server encrypt at once, which collides in the + * TLS half-duplex flow; the direct single-op tests drive decrypt on + * its own instead. */ + if (info->cipher.enc == 0 && !hsm->offloadDec) + return WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE); + + XMEMSET(j, 0, sizeof(*j)); + j->dec = (info->cipher.enc == 0); + + switch (info->cipher.type) { + #ifdef HAVE_AESGCM + case WC_CIPHER_AES_GCM: + if (j->dec) { + wc_CryptoCb_AesAuthDec* e = &info->cipher.aesgcm_dec; + j->obj = e->aes; j->out = e->out; j->in = e->in; j->sz = e->sz; + j->iv = e->iv; j->ivSz = e->ivSz; + j->authTagDec = e->authTag; j->authTagSz = e->authTagSz; + j->authIn = e->authIn; j->authInSz = e->authInSz; + } + else { + wc_CryptoCb_AesAuthEnc* e = &info->cipher.aesgcm_enc; + j->obj = e->aes; j->out = e->out; j->in = e->in; j->sz = e->sz; + j->iv = e->iv; j->ivSz = e->ivSz; + j->authTag = e->authTag; j->authTagSz = e->authTagSz; + j->authIn = e->authIn; j->authInSz = e->authInSz; + } + break; + #endif + #ifdef HAVE_AESCCM + case WC_CIPHER_AES_CCM: /* CCM uses the nonce field */ + if (j->dec) { + wc_CryptoCb_AesAuthDec* e = &info->cipher.aesccm_dec; + j->obj = e->aes; j->out = e->out; j->in = e->in; j->sz = e->sz; + j->iv = e->nonce; j->ivSz = e->nonceSz; + j->authTagDec = e->authTag; j->authTagSz = e->authTagSz; + j->authIn = e->authIn; j->authInSz = e->authInSz; + } + else { + wc_CryptoCb_AesAuthEnc* e = &info->cipher.aesccm_enc; + j->obj = e->aes; j->out = e->out; j->in = e->in; j->sz = e->sz; + j->iv = e->nonce; j->ivSz = e->nonceSz; + j->authTag = e->authTag; j->authTagSz = e->authTagSz; + j->authIn = e->authIn; j->authInSz = e->authInSz; + } + break; + #endif + #ifdef HAVE_AES_CBC + case WC_CIPHER_AES_CBC: + j->obj = info->cipher.aescbc.aes; j->out = info->cipher.aescbc.out; + j->in = info->cipher.aescbc.in; j->sz = info->cipher.aescbc.sz; + break; + #endif + #ifndef NO_DES3 + case WC_CIPHER_DES3: + j->obj = info->cipher.des3.des; j->out = info->cipher.des3.out; + j->in = info->cipher.des3.in; j->sz = info->cipher.des3.sz; + break; + #endif + default: + /* not a record cipher this HSM offloads: run in software */ + return WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE); + } + + j->active = 1; + j->type = info->cipher.type; + j->pendRemaining = hsm->pendPolls; + hsm->submits++; + if (j->dec) + hsm->decSubmits++; + return WC_PENDING_E; + } + + if (info->algo_type == WC_ALGO_TYPE_ASYNC_POLL) { + TestHsmJob* j = &hsm->job; + hsm->polls++; + if (!j->active) + return WC_NO_ERR_TRACE(WC_NO_PENDING_E); + if (j->pendRemaining > 0) { + j->pendRemaining--; /* still in-flight this poll */ + return WC_PENDING_E; + } + j->active = 0; + return test_hsm_complete(j); + } + + return WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE); +} + +/* Reset the device counters and in-flight job for another run, and set how + * many polls the next op stays in-flight (pendPolls) before completing. */ +static void test_hsm_reset(TestHsm* hsm, int pendPolls) +{ + XMEMSET(&hsm->job, 0, sizeof(hsm->job)); + hsm->pendPolls = pendPolls; + hsm->invocations = 0; + hsm->submits = 0; + hsm->decSubmits = 0; + hsm->polls = 0; +} + +/* Init the event and drive the poll to completion, bounded so a missing poll + * path fails instead of hanging. Returns the event result. */ +static int test_async_drive_poll(WC_ASYNC_DEV* dev, int ret) +{ + if (ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { + WOLF_EVENT* ev = &dev->event; + int i; + (void)wolfAsync_EventInit(ev, WOLF_EVENT_TYPE_ASYNC_WOLFCRYPT, dev, + WC_ASYNC_FLAG_NONE); + for (i = 0; i < TEST_ASYNC_MAX_POLLS && + ev->ret == WC_NO_ERR_TRACE(WC_PENDING_E); i++) { + (void)wolfAsync_EventPoll(ev, WOLF_POLL_FLAG_CHECK_HW); + } + ret = ev->ret; + } + return ret; +} + +static WC_INLINE const byte* test_async_key32(void) +{ + static const byte k[32] = { + 0x60,0x3d,0xeb,0x10,0x15,0xca,0x71,0xbe, + 0x2b,0x73,0xae,0xf0,0x85,0x7d,0x77,0x81, + 0x1f,0x35,0x2c,0x07,0x3b,0x61,0x08,0xd7, + 0x2d,0x98,0x10,0xa3,0x09,0x14,0xdf,0xf4 + }; + return k; +} +static WC_INLINE const byte* test_async_iv12(void) +{ + static const byte v[12] = { + 0xca,0xfe,0xba,0xbe,0xfa,0xce,0xdb,0xad,0xde,0xca,0xf8,0x88 + }; + return v; +} +static WC_INLINE const byte* test_async_iv16(void) +{ + static const byte v[16] = { + 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07, + 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f + }; + return v; +} +static WC_INLINE const byte* test_async_plain32(void) +{ + static const byte p[32] = { + 0xd9,0x31,0x32,0x25,0xf8,0x84,0x06,0xe5, + 0xa5,0x59,0x09,0xc5,0xaf,0xf5,0x26,0x9a, + 0x86,0xa7,0xa9,0x53,0x15,0x34,0xf7,0xda, + 0x2e,0x4c,0x30,0x3d,0x8a,0x31,0x8a,0x72 + }; + return p; +} +static WC_INLINE const byte* test_async_aad12(void) +{ + static const byte a[12] = { + 0xfe,0xed,0xfa,0xce,0xde,0xad,0xbe,0xef,0xfe,0xed,0xfa,0xce + }; + return a; +} + +#endif /* feature macros */ + +/* + * AES-GCM record cipher: pends, poll-completes, and produces the same + * ciphertext and tag as a pure software encrypt. + */ +int test_wc_CryptoCb_AsyncPollAesGcm(void) +{ + EXPECT_DECLS; +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) && defined(HAVE_AESGCM) + TestHsm hsm; + Aes hw, sw; + byte out[32], ref[32], tag[16], rtag[16]; + int registered = 0; + int pend; + + XMEMSET(&hsm, 0, sizeof(hsm)); + XMEMSET(&hw, 0, sizeof(hw)); + XMEMSET(&sw, 0, sizeof(sw)); + + ExpectIntEQ(wc_CryptoCb_RegisterDevice(TEST_ASYNC_HSM_DEVID, test_hsm_cb, + &hsm), 0); + if (EXPECT_SUCCESS()) + registered = 1; + + ExpectIntEQ(wc_AesInit(&hw, NULL, TEST_ASYNC_HSM_DEVID), 0); + ExpectIntEQ(wc_AesInit(&sw, NULL, INVALID_DEVID), 0); + ExpectIntEQ(wc_AesGcmSetKey(&hw, test_async_key32(), 32), 0); + ExpectIntEQ(wc_AesGcmSetKey(&sw, test_async_key32(), 32), 0); + + /* software reference */ + ExpectIntEQ(wc_AesGcmEncrypt(&sw, ref, test_async_plain32(), 32, + test_async_iv12(), 12, rtag, 16, test_async_aad12(), 12), 0); + + /* pend depth 0..TEST_ASYNC_MAX_PEND: op completes on poll number pend+1 */ + for (pend = 0; pend <= TEST_ASYNC_MAX_PEND; pend++) { + test_hsm_reset(&hsm, pend); + ExpectIntEQ(test_async_drive_poll(&hw.asyncDev, + wc_AesGcmEncrypt(&hw, out, test_async_plain32(), 32, + test_async_iv12(), 12, tag, 16, test_async_aad12(), 12)), 0); + ExpectIntEQ(hsm.submits, 1); + ExpectIntEQ(hsm.polls, pend + 1); + ExpectBufEQ(out, ref, 32); + ExpectBufEQ(tag, rtag, 16); + } + + /* decrypt side: poll-complete recovers the plaintext (same GCM key) */ + hsm.offloadDec = 1; + for (pend = 0; pend <= TEST_ASYNC_MAX_PEND; pend++) { + test_hsm_reset(&hsm, pend); + ExpectIntEQ(test_async_drive_poll(&hw.asyncDev, + wc_AesGcmDecrypt(&hw, out, ref, 32, + test_async_iv12(), 12, rtag, 16, test_async_aad12(), 12)), 0); + ExpectIntEQ(hsm.submits, 1); + ExpectIntEQ(hsm.polls, pend + 1); + ExpectBufEQ(out, test_async_plain32(), 32); + } + + wc_AesFree(&hw); + wc_AesFree(&sw); + if (registered) + wc_CryptoCb_UnRegisterDevice(TEST_ASYNC_HSM_DEVID); +#endif + return EXPECT_RESULT(); +} + +/* + * AES-CBC record cipher: pends, poll-completes, matches software. + */ +int test_wc_CryptoCb_AsyncPollAesCbc(void) +{ + EXPECT_DECLS; +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) && defined(HAVE_AES_CBC) + TestHsm hsm; + Aes hw, sw, hwd; + byte out[32], ref[32]; + int registered = 0; + int pend; + + XMEMSET(&hsm, 0, sizeof(hsm)); + XMEMSET(&hw, 0, sizeof(hw)); + XMEMSET(&sw, 0, sizeof(sw)); + XMEMSET(&hwd, 0, sizeof(hwd)); + + ExpectIntEQ(wc_CryptoCb_RegisterDevice(TEST_ASYNC_HSM_DEVID, test_hsm_cb, + &hsm), 0); + if (EXPECT_SUCCESS()) + registered = 1; + + ExpectIntEQ(wc_AesInit(&hw, NULL, TEST_ASYNC_HSM_DEVID), 0); + ExpectIntEQ(wc_AesInit(&sw, NULL, INVALID_DEVID), 0); + ExpectIntEQ(wc_AesInit(&hwd, NULL, TEST_ASYNC_HSM_DEVID), 0); + ExpectIntEQ(wc_AesSetKey(&hw, test_async_key32(), 32, test_async_iv16(), + AES_ENCRYPTION), 0); + ExpectIntEQ(wc_AesSetKey(&sw, test_async_key32(), 32, test_async_iv16(), + AES_ENCRYPTION), 0); + ExpectIntEQ(wc_AesSetKey(&hwd, test_async_key32(), 32, test_async_iv16(), + AES_DECRYPTION), 0); + + ExpectIntEQ(wc_AesCbcEncrypt(&sw, ref, test_async_plain32(), 32), 0); + + for (pend = 0; pend <= TEST_ASYNC_MAX_PEND; pend++) { + test_hsm_reset(&hsm, pend); + /* CBC chains, so re-set the IV before each encrypt */ + ExpectIntEQ(wc_AesSetIV(&hw, test_async_iv16()), 0); + ExpectIntEQ(test_async_drive_poll(&hw.asyncDev, + wc_AesCbcEncrypt(&hw, out, test_async_plain32(), 32)), 0); + ExpectIntEQ(hsm.submits, 1); + ExpectIntEQ(hsm.polls, pend + 1); + ExpectBufEQ(out, ref, 32); + } + + /* decrypt side: poll-complete recovers the plaintext from ref */ + hsm.offloadDec = 1; + for (pend = 0; pend <= TEST_ASYNC_MAX_PEND; pend++) { + test_hsm_reset(&hsm, pend); + ExpectIntEQ(wc_AesSetIV(&hwd, test_async_iv16()), 0); + ExpectIntEQ(test_async_drive_poll(&hwd.asyncDev, + wc_AesCbcDecrypt(&hwd, out, ref, 32)), 0); + ExpectIntEQ(hsm.submits, 1); + ExpectIntEQ(hsm.polls, pend + 1); + ExpectBufEQ(out, test_async_plain32(), 32); + } + + wc_AesFree(&hw); + wc_AesFree(&sw); + wc_AesFree(&hwd); + if (registered) + wc_CryptoCb_UnRegisterDevice(TEST_ASYNC_HSM_DEVID); +#endif + return EXPECT_RESULT(); +} + +/* + * AES-CCM record cipher: pends, poll-completes, matches software. + */ +int test_wc_CryptoCb_AsyncPollAesCcm(void) +{ + EXPECT_DECLS; +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) && defined(HAVE_AESCCM) + TestHsm hsm; + Aes hw, sw; + byte out[32], ref[32], tag[16], rtag[16]; + int registered = 0; + int pend; + + XMEMSET(&hsm, 0, sizeof(hsm)); + XMEMSET(&hw, 0, sizeof(hw)); + XMEMSET(&sw, 0, sizeof(sw)); + + ExpectIntEQ(wc_CryptoCb_RegisterDevice(TEST_ASYNC_HSM_DEVID, test_hsm_cb, + &hsm), 0); + if (EXPECT_SUCCESS()) + registered = 1; + + ExpectIntEQ(wc_AesInit(&hw, NULL, TEST_ASYNC_HSM_DEVID), 0); + ExpectIntEQ(wc_AesInit(&sw, NULL, INVALID_DEVID), 0); + ExpectIntEQ(wc_AesCcmSetKey(&hw, test_async_key32(), 16), 0); + ExpectIntEQ(wc_AesCcmSetKey(&sw, test_async_key32(), 16), 0); + + ExpectIntEQ(wc_AesCcmEncrypt(&sw, ref, test_async_plain32(), 32, + test_async_iv12(), 12, rtag, 16, test_async_aad12(), 12), 0); + + for (pend = 0; pend <= TEST_ASYNC_MAX_PEND; pend++) { + test_hsm_reset(&hsm, pend); + ExpectIntEQ(test_async_drive_poll(&hw.asyncDev, + wc_AesCcmEncrypt(&hw, out, test_async_plain32(), 32, + test_async_iv12(), 12, tag, 16, test_async_aad12(), 12)), 0); + ExpectIntEQ(hsm.submits, 1); + ExpectIntEQ(hsm.polls, pend + 1); + ExpectBufEQ(out, ref, 32); + ExpectBufEQ(tag, rtag, 16); + } + + /* decrypt side: poll-complete recovers the plaintext (same CCM key) */ + hsm.offloadDec = 1; + for (pend = 0; pend <= TEST_ASYNC_MAX_PEND; pend++) { + test_hsm_reset(&hsm, pend); + ExpectIntEQ(test_async_drive_poll(&hw.asyncDev, + wc_AesCcmDecrypt(&hw, out, ref, 32, + test_async_iv12(), 12, rtag, 16, test_async_aad12(), 12)), 0); + ExpectIntEQ(hsm.submits, 1); + ExpectIntEQ(hsm.polls, pend + 1); + ExpectBufEQ(out, test_async_plain32(), 32); + } + + wc_AesFree(&hw); + wc_AesFree(&sw); + if (registered) + wc_CryptoCb_UnRegisterDevice(TEST_ASYNC_HSM_DEVID); +#endif + return EXPECT_RESULT(); +} + +/* + * 3DES-CBC record cipher: pends, poll-completes, matches software. Covers the + * WOLFSSL_ASYNC_MARKER_3DES routing (not just AES). + */ +int test_wc_CryptoCb_AsyncPollDes3(void) +{ + EXPECT_DECLS; +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) && !defined(NO_DES3) + TestHsm hsm; + Des3 hw, sw, hwd; + byte out[32], ref[32]; + static const byte key24[24] = { + 0x01,0x23,0x45,0x67,0x89,0xab,0xcd,0xef, + 0x23,0x45,0x67,0x89,0xab,0xcd,0xef,0x01, + 0x45,0x67,0x89,0xab,0xcd,0xef,0x01,0x23 + }; + static const byte iv8[8] = { + 0x12,0x34,0x56,0x78,0x90,0xab,0xcd,0xef + }; + int registered = 0; + int pend; + + XMEMSET(&hsm, 0, sizeof(hsm)); + XMEMSET(&hw, 0, sizeof(hw)); + XMEMSET(&sw, 0, sizeof(sw)); + XMEMSET(&hwd, 0, sizeof(hwd)); + + ExpectIntEQ(wc_CryptoCb_RegisterDevice(TEST_ASYNC_HSM_DEVID, test_hsm_cb, + &hsm), 0); + if (EXPECT_SUCCESS()) + registered = 1; + + ExpectIntEQ(wc_Des3Init(&hw, NULL, TEST_ASYNC_HSM_DEVID), 0); + ExpectIntEQ(wc_Des3Init(&sw, NULL, INVALID_DEVID), 0); + ExpectIntEQ(wc_Des3Init(&hwd, NULL, TEST_ASYNC_HSM_DEVID), 0); + ExpectIntEQ(wc_Des3_SetKey(&hw, key24, iv8, DES_ENCRYPTION), 0); + ExpectIntEQ(wc_Des3_SetKey(&sw, key24, iv8, DES_ENCRYPTION), 0); + ExpectIntEQ(wc_Des3_SetKey(&hwd, key24, iv8, DES_DECRYPTION), 0); + + ExpectIntEQ(wc_Des3_CbcEncrypt(&sw, ref, test_async_plain32(), 32), 0); + + for (pend = 0; pend <= TEST_ASYNC_MAX_PEND; pend++) { + test_hsm_reset(&hsm, pend); + /* CBC chains, so re-set the IV before each encrypt */ + ExpectIntEQ(wc_Des3_SetIV(&hw, iv8), 0); + ExpectIntEQ(test_async_drive_poll(&hw.asyncDev, + wc_Des3_CbcEncrypt(&hw, out, test_async_plain32(), 32)), 0); + ExpectIntEQ(hsm.submits, 1); + ExpectIntEQ(hsm.polls, pend + 1); + ExpectBufEQ(out, ref, 32); + } + + /* decrypt side: poll-complete recovers the plaintext from ref */ + hsm.offloadDec = 1; + for (pend = 0; pend <= TEST_ASYNC_MAX_PEND; pend++) { + test_hsm_reset(&hsm, pend); + ExpectIntEQ(wc_Des3_SetIV(&hwd, iv8), 0); + ExpectIntEQ(test_async_drive_poll(&hwd.asyncDev, + wc_Des3_CbcDecrypt(&hwd, out, ref, 32)), 0); + ExpectIntEQ(hsm.submits, 1); + ExpectIntEQ(hsm.polls, pend + 1); + ExpectBufEQ(out, test_async_plain32(), 32); + } + + wc_Des3Free(&hw); + wc_Des3Free(&sw); + wc_Des3Free(&hwd); + if (registered) + wc_CryptoCb_UnRegisterDevice(TEST_ASYNC_HSM_DEVID); +#endif + return EXPECT_RESULT(); +} + +/* + * A cipher with a crypto callback dispatch that the HSM declines (AES-CTR): + * the callback IS entered (it has a dispatch) but returns unavailable, so the + * op falls back to software with no pend and no poll. Contrast with the + * unimplemented ciphers below, where the callback is never entered. + */ +int test_wc_CryptoCb_AsyncPollUnsupported(void) +{ + EXPECT_DECLS; +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) && defined(WOLFSSL_AES_COUNTER) + TestHsm hsm; + Aes hw, sw; + byte out[32], ref[32]; + int registered = 0; + + XMEMSET(&hsm, 0, sizeof(hsm)); + XMEMSET(&hw, 0, sizeof(hw)); + XMEMSET(&sw, 0, sizeof(sw)); + + ExpectIntEQ(wc_CryptoCb_RegisterDevice(TEST_ASYNC_HSM_DEVID, test_hsm_cb, + &hsm), 0); + if (EXPECT_SUCCESS()) + registered = 1; + + ExpectIntEQ(wc_AesInit(&hw, NULL, TEST_ASYNC_HSM_DEVID), 0); + ExpectIntEQ(wc_AesInit(&sw, NULL, INVALID_DEVID), 0); + ExpectIntEQ(wc_AesSetKey(&hw, test_async_key32(), 32, test_async_iv16(), + AES_ENCRYPTION), 0); + ExpectIntEQ(wc_AesSetKey(&sw, test_async_key32(), 32, test_async_iv16(), + AES_ENCRYPTION), 0); + + /* declined by the HSM -> software fallback, completes synchronously */ + ExpectIntEQ(wc_AesCtrEncrypt(&hw, out, test_async_plain32(), 32), 0); + ExpectIntGE(hsm.invocations, 1); /* dispatch exists: callback was entered */ + ExpectIntEQ(hsm.submits, 0); /* but declined: not offloaded */ + ExpectIntEQ(hsm.polls, 0); + + ExpectIntEQ(wc_AesCtrEncrypt(&sw, ref, test_async_plain32(), 32), 0); + ExpectBufEQ(out, ref, 32); + + wc_AesFree(&hw); + wc_AesFree(&sw); + if (registered) + wc_CryptoCb_UnRegisterDevice(TEST_ASYNC_HSM_DEVID); +#endif + return EXPECT_RESULT(); +} + +/* + * Negative test: ChaCha20-Poly1305 is named in the crypto callback cipher type + * table (WC_CIPHER_CHACHA) but has no dispatch, so it is NOT offloadable. With + * a device registered, a ChaCha20-Poly1305 encrypt must never enter the + * callback and must run entirely in software. If crypto callback ChaCha + * support is ever added, this test will fail and must be updated. + */ +int test_wc_CryptoCb_AsyncPollChachaUnimpl(void) +{ + EXPECT_DECLS; +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) && defined(HAVE_CHACHA) && \ + defined(HAVE_POLY1305) + TestHsm hsm; + byte out[32], tag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE]; + static const byte key[CHACHA20_POLY1305_AEAD_KEYSIZE] = { + 0x80,0x81,0x82,0x83,0x84,0x85,0x86,0x87, + 0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f, + 0x90,0x91,0x92,0x93,0x94,0x95,0x96,0x97, + 0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f + }; + static const byte iv[CHACHA20_POLY1305_AEAD_IV_SIZE] = { + 0x07,0x00,0x00,0x00,0x40,0x41,0x42,0x43,0x44,0x45,0x46,0x47 + }; + int registered = 0; + + XMEMSET(&hsm, 0, sizeof(hsm)); + + ExpectIntEQ(wc_CryptoCb_RegisterDevice(TEST_ASYNC_HSM_DEVID, test_hsm_cb, + &hsm), 0); + if (EXPECT_SUCCESS()) + registered = 1; + + /* no crypto callback dispatch for ChaCha: runs in software, cb untouched */ + ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt(key, iv, + test_async_aad12(), 12, test_async_plain32(), 32, out, tag), 0); + ExpectIntEQ(hsm.invocations, 0); + ExpectIntEQ(hsm.submits, 0); + ExpectIntEQ(hsm.polls, 0); + + if (registered) + wc_CryptoCb_UnRegisterDevice(TEST_ASYNC_HSM_DEVID); +#endif + return EXPECT_RESULT(); +} + +/* + * Negative test: single DES is named in the crypto callback cipher type table + * (WC_CIPHER_DES) but has no dispatch, so a DES-CBC encrypt with a device + * registered must never enter the callback and run entirely in software. + */ +int test_wc_CryptoCb_AsyncPollDesUnimpl(void) +{ + EXPECT_DECLS; +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) && !defined(NO_DES3) + TestHsm hsm; + Des des; + byte out[32]; + static const byte key8[8] = { + 0x01,0x23,0x45,0x67,0x89,0xab,0xcd,0xef + }; + static const byte iv8[8] = { + 0x12,0x34,0x56,0x78,0x90,0xab,0xcd,0xef + }; + int registered = 0; + + XMEMSET(&hsm, 0, sizeof(hsm)); + XMEMSET(&des, 0, sizeof(des)); + + ExpectIntEQ(wc_CryptoCb_RegisterDevice(TEST_ASYNC_HSM_DEVID, test_hsm_cb, + &hsm), 0); + if (EXPECT_SUCCESS()) + registered = 1; + + ExpectIntEQ(wc_Des_SetKey(&des, key8, iv8, DES_ENCRYPTION), 0); + /* no crypto callback dispatch for single DES: cb untouched */ + ExpectIntEQ(wc_Des_CbcEncrypt(&des, out, test_async_plain32(), 32), 0); + ExpectIntEQ(hsm.invocations, 0); + ExpectIntEQ(hsm.submits, 0); + ExpectIntEQ(hsm.polls, 0); + + if (registered) + wc_CryptoCb_UnRegisterDevice(TEST_ASYNC_HSM_DEVID); +#endif + return EXPECT_RESULT(); +} + +/* + * Full-stack TLS 1.3 tests. One generic harness drives an in-memory + * client/server handshake and an application-data echo through the crypto + * callback record cipher; only the callback implementation and the pinned + * ciphersuite are swapped per test. + */ +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) && defined(WOLFSSL_TLS13) && \ + defined(HAVE_AESGCM) && defined(HAVE_MANUAL_MEMIO_TESTS_DEPENDENCIES) && \ + !defined(NO_WOLFSSL_CLIENT) && !defined(NO_WOLFSSL_SERVER) + +#define TEST_ASYNC_TLS + +/* A device that offloads the AES-GCM record cipher but does NOT implement the + * poll completion (returns unavailable for WC_ALGO_TYPE_ASYNC_POLL). The op + * pends and can never finish, so the record layer must fail rather than ship + * an unencrypted record. */ +static int test_hsm_nopoll_cb(int devId, wc_CryptoInfo* info, void* ctx) +{ + TestHsm* hsm = (TestHsm*)ctx; + (void)devId; + + if (info == NULL) + return WC_NO_ERR_TRACE(BAD_FUNC_ARG); + + hsm->invocations++; + + if (info->algo_type == WC_ALGO_TYPE_CIPHER && info->cipher.enc == 1 && + info->cipher.type == WC_CIPHER_AES_GCM) { + hsm->submits++; + return WC_PENDING_E; /* pended, but no poll support */ + } + if (info->algo_type == WC_ALGO_TYPE_ASYNC_POLL) { + hsm->polls++; + return WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE); + } + return WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE); +} + +/* Drive one wolfSSL_write/read through, servicing async pends. Returns 1 if + * the message round-trips, 0 on failure. */ +static int test_async_tls_echo(WOLFSSL* ssl_c, WOLFSSL* ssl_s) +{ + const char msg[] = "hello async record layer"; + byte buf[64]; + int ret, err, i; + + for (i = 0, ret = -1; i < 40; i++) { + ret = wolfSSL_write(ssl_c, msg, (int)sizeof(msg)); + if (ret == (int)sizeof(msg)) + break; + err = wolfSSL_get_error(ssl_c, ret); + if (err == WC_NO_ERR_TRACE(WC_PENDING_E)) + (void)wolfSSL_AsyncPoll(ssl_c, WOLF_POLL_FLAG_CHECK_HW); + else if (err != WOLFSSL_ERROR_WANT_READ && + err != WOLFSSL_ERROR_WANT_WRITE) + return 0; + } + if (ret != (int)sizeof(msg)) + return 0; + + for (i = 0, ret = -1; i < 40; i++) { + ret = wolfSSL_read(ssl_s, buf, (int)sizeof(buf)); + if (ret > 0) + break; + err = wolfSSL_get_error(ssl_s, ret); + if (err == WC_NO_ERR_TRACE(WC_PENDING_E)) + (void)wolfSSL_AsyncPoll(ssl_s, WOLF_POLL_FLAG_CHECK_HW); + else if (err != WOLFSSL_ERROR_WANT_READ && + err != WOLFSSL_ERROR_WANT_WRITE) + return 0; + } + return (ret == (int)sizeof(msg) && XMEMCMP(buf, msg, sizeof(msg)) == 0); +} + +typedef struct TlsAsyncResult { + int handshake; /* 0 = handshake completed */ + int dataOk; /* 1 = echo round-tripped */ + int cipherSubmits; /* record ciphers offloaded to the device */ +} TlsAsyncResult; + +/* Generic harness: register the given callback, run a TLS 1.3 handshake and + * echo over memio with the given ciphersuite pinned, and report the outcome + * plus how many record ciphers were offloaded. */ +static void test_async_tls_run(CryptoDevCallbackFunc cb, const char* cipherList, + int pendPolls, TlsAsyncResult* res) +{ + WOLFSSL_CTX* ctx_c = NULL; + WOLFSSL_CTX* ctx_s = NULL; + WOLFSSL* ssl_c = NULL; + WOLFSSL* ssl_s = NULL; + struct test_memio_ctx test_ctx; + TestHsm hsm; + int registered = 0; + + res->handshake = -1; + res->dataOk = 0; + res->cipherSubmits = 0; + + XMEMSET(&test_ctx, 0, sizeof(test_ctx)); + XMEMSET(&hsm, 0, sizeof(hsm)); + hsm.pendPolls = pendPolls; /* polls each record cipher stays in-flight */ + + if (wc_CryptoCb_RegisterDevice(TEST_ASYNC_HSM_DEVID, cb, &hsm) != 0) + return; + registered = 1; + + if (test_memio_setup(&test_ctx, &ctx_c, &ctx_s, &ssl_c, &ssl_s, + wolfTLSv1_3_client_method, wolfTLSv1_3_server_method) != 0) + goto cleanup; + + (void)wolfSSL_CTX_SetDevId(ctx_c, TEST_ASYNC_HSM_DEVID); + (void)wolfSSL_CTX_SetDevId(ctx_s, TEST_ASYNC_HSM_DEVID); + (void)wolfSSL_SetDevId(ssl_c, TEST_ASYNC_HSM_DEVID); + (void)wolfSSL_SetDevId(ssl_s, TEST_ASYNC_HSM_DEVID); + + if (cipherList != NULL) { + if (wolfSSL_set_cipher_list(ssl_c, cipherList) != WOLFSSL_SUCCESS || + wolfSSL_set_cipher_list(ssl_s, cipherList) != WOLFSSL_SUCCESS) + goto cleanup; + } + + res->handshake = test_memio_do_handshake(ssl_c, ssl_s, 10, NULL); + if (res->handshake == 0) + res->dataOk = test_async_tls_echo(ssl_c, ssl_s); + res->cipherSubmits = hsm.submits; + +cleanup: + if (ssl_c != NULL) + wolfSSL_free(ssl_c); + if (ssl_s != NULL) + wolfSSL_free(ssl_s); + if (ctx_c != NULL) + wolfSSL_CTX_free(ctx_c); + if (ctx_s != NULL) + wolfSSL_CTX_free(ctx_s); + if (registered) + wc_CryptoCb_UnRegisterDevice(TEST_ASYNC_HSM_DEVID); +} + +#endif /* TEST_ASYNC_TLS */ + +/* + * Full TLS 1.3 with AES-GCM offloaded to a poll-completing device: the + * handshake and an application-data echo both succeed, and the record cipher + * is offloaded (poll completion drives the encrypted records). + */ +int test_wc_CryptoCb_AsyncPollTlsAesGcm(void) +{ + EXPECT_DECLS; +#ifdef TEST_ASYNC_TLS + TlsAsyncResult res; + int pend; + + /* pend depth 0..TEST_ASYNC_MAX_PEND: each record cipher stays in-flight for + * that many extra polls before completing. */ + for (pend = 0; pend <= TEST_ASYNC_MAX_PEND; pend++) { + test_async_tls_run(test_hsm_cb, + "TLS13-AES128-GCM-SHA256:TLS13-AES256-GCM-SHA384", pend, &res); + + ExpectIntEQ(res.handshake, 0); + ExpectIntEQ(res.dataOk, 1); + ExpectIntGT(res.cipherSubmits, 0); + } +#endif + return EXPECT_RESULT(); +} + +/* + * Full TLS 1.3 with ChaCha20-Poly1305: ChaCha has no crypto callback dispatch, + * so the record cipher runs in software. The handshake and echo still succeed + * and the device is never asked to offload a record cipher. + */ +int test_wc_CryptoCb_AsyncPollTlsChachaNotOffloaded(void) +{ + EXPECT_DECLS; +#if defined(TEST_ASYNC_TLS) && defined(HAVE_CHACHA) && defined(HAVE_POLY1305) + TlsAsyncResult res; + + test_async_tls_run(test_hsm_cb, "TLS13-CHACHA20-POLY1305-SHA256", 1, &res); + + ExpectIntEQ(res.handshake, 0); + ExpectIntEQ(res.dataOk, 1); + ExpectIntEQ(res.cipherSubmits, 0); /* ChaCha not dispatched to the device */ +#endif + return EXPECT_RESULT(); +} + +/* + * Full TLS 1.3 with AES-GCM offloaded to a device that pends the record cipher + * but does NOT implement poll completion: the handshake must fail rather than + * transmit an unencrypted record. + */ +int test_wc_CryptoCb_AsyncPollTlsNoPollFails(void) +{ + EXPECT_DECLS; +#ifdef TEST_ASYNC_TLS + TlsAsyncResult res; + + test_async_tls_run(test_hsm_nopoll_cb, + "TLS13-AES128-GCM-SHA256:TLS13-AES256-GCM-SHA384", 1, &res); + + ExpectIntNE(res.handshake, 0); /* expected failure */ + ExpectIntEQ(res.dataOk, 0); + ExpectIntGT(res.cipherSubmits, 0); /* it did pend a record cipher */ +#endif + return EXPECT_RESULT(); +} + +/* + * Full TLS 1.3 with AES-GCM offloaded on BOTH the encrypt and decrypt record + * paths. Each peer uses its own poll-completing device, so a client decrypt and + * a server encrypt never share one job slot. The handshake and echo succeed and + * both peers offload at least one decrypt through poll completion, confirming + * the poll model drives the receive (decrypt) side, not just send. + */ +int test_wc_CryptoCb_AsyncPollTlsBothDirections(void) +{ + EXPECT_DECLS; +#ifdef TEST_ASYNC_TLS + WOLFSSL_CTX* ctx_c = NULL; + WOLFSSL_CTX* ctx_s = NULL; + WOLFSSL* ssl_c = NULL; + WOLFSSL* ssl_s = NULL; + struct test_memio_ctx test_ctx; + TestHsm hsm_c, hsm_s; + int reg_c = 0, reg_s = 0; + int handshake = -1, dataOk = 0; + const char* ciphers = "TLS13-AES128-GCM-SHA256:TLS13-AES256-GCM-SHA384"; + + XMEMSET(&test_ctx, 0, sizeof(test_ctx)); + XMEMSET(&hsm_c, 0, sizeof(hsm_c)); + XMEMSET(&hsm_s, 0, sizeof(hsm_s)); + hsm_c.pendPolls = 1; hsm_c.offloadDec = 1; + hsm_s.pendPolls = 1; hsm_s.offloadDec = 1; + + ExpectIntEQ(wc_CryptoCb_RegisterDevice(TEST_ASYNC_HSM_DEVID, test_hsm_cb, + &hsm_c), 0); + if (EXPECT_SUCCESS()) + reg_c = 1; + ExpectIntEQ(wc_CryptoCb_RegisterDevice(TEST_ASYNC_HSM_DEVID2, test_hsm_cb, + &hsm_s), 0); + if (EXPECT_SUCCESS()) + reg_s = 1; + + ExpectIntEQ(test_memio_setup(&test_ctx, &ctx_c, &ctx_s, &ssl_c, &ssl_s, + wolfTLSv1_3_client_method, wolfTLSv1_3_server_method), 0); + + if (ssl_c != NULL && ssl_s != NULL) { + (void)wolfSSL_CTX_SetDevId(ctx_c, TEST_ASYNC_HSM_DEVID); + (void)wolfSSL_CTX_SetDevId(ctx_s, TEST_ASYNC_HSM_DEVID2); + (void)wolfSSL_SetDevId(ssl_c, TEST_ASYNC_HSM_DEVID); + (void)wolfSSL_SetDevId(ssl_s, TEST_ASYNC_HSM_DEVID2); + + ExpectIntEQ(wolfSSL_set_cipher_list(ssl_c, ciphers), WOLFSSL_SUCCESS); + ExpectIntEQ(wolfSSL_set_cipher_list(ssl_s, ciphers), WOLFSSL_SUCCESS); + + /* Both directions offloaded on both peers means many more poll cycles + * than the encrypt-only tests, so allow more handshake rounds. */ + handshake = test_memio_do_handshake(ssl_c, ssl_s, 64, NULL); + ExpectIntEQ(handshake, 0); + if (handshake == 0) + dataOk = test_async_tls_echo(ssl_c, ssl_s); + ExpectIntEQ(dataOk, 1); + + /* both peers offloaded record ciphers, including >=1 decrypt each */ + ExpectIntGT(hsm_c.submits, 0); + ExpectIntGT(hsm_s.submits, 0); + ExpectIntGT(hsm_c.decSubmits, 0); + ExpectIntGT(hsm_s.decSubmits, 0); + } + + wolfSSL_free(ssl_c); + wolfSSL_free(ssl_s); + wolfSSL_CTX_free(ctx_c); + wolfSSL_CTX_free(ctx_s); + if (reg_c) + wc_CryptoCb_UnRegisterDevice(TEST_ASYNC_HSM_DEVID); + if (reg_s) + wc_CryptoCb_UnRegisterDevice(TEST_ASYNC_HSM_DEVID2); +#endif + return EXPECT_RESULT(); +} diff --git a/tests/api/test_async.h b/tests/api/test_async.h new file mode 100644 index 00000000000..ba5b8219533 --- /dev/null +++ b/tests/api/test_async.h @@ -0,0 +1,53 @@ +/* test_async.h + * + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#ifndef WOLFCRYPT_TEST_ASYNC_H +#define WOLFCRYPT_TEST_ASYNC_H + +#include + +int test_wc_CryptoCb_AsyncPollAesGcm(void); +int test_wc_CryptoCb_AsyncPollAesCbc(void); +int test_wc_CryptoCb_AsyncPollAesCcm(void); +int test_wc_CryptoCb_AsyncPollDes3(void); +int test_wc_CryptoCb_AsyncPollUnsupported(void); +int test_wc_CryptoCb_AsyncPollChachaUnimpl(void); +int test_wc_CryptoCb_AsyncPollDesUnimpl(void); +int test_wc_CryptoCb_AsyncPollTlsAesGcm(void); +int test_wc_CryptoCb_AsyncPollTlsChachaNotOffloaded(void); +int test_wc_CryptoCb_AsyncPollTlsNoPollFails(void); +int test_wc_CryptoCb_AsyncPollTlsBothDirections(void); + +#define TEST_ASYNC_DECLS \ + TEST_DECL_GROUP("async", test_wc_CryptoCb_AsyncPollAesGcm), \ + TEST_DECL_GROUP("async", test_wc_CryptoCb_AsyncPollAesCbc), \ + TEST_DECL_GROUP("async", test_wc_CryptoCb_AsyncPollAesCcm), \ + TEST_DECL_GROUP("async", test_wc_CryptoCb_AsyncPollDes3), \ + TEST_DECL_GROUP("async", test_wc_CryptoCb_AsyncPollUnsupported), \ + TEST_DECL_GROUP("async", test_wc_CryptoCb_AsyncPollChachaUnimpl), \ + TEST_DECL_GROUP("async", test_wc_CryptoCb_AsyncPollDesUnimpl), \ + TEST_DECL_GROUP("async", test_wc_CryptoCb_AsyncPollTlsAesGcm), \ + TEST_DECL_GROUP("async", \ + test_wc_CryptoCb_AsyncPollTlsChachaNotOffloaded), \ + TEST_DECL_GROUP("async", test_wc_CryptoCb_AsyncPollTlsNoPollFails), \ + TEST_DECL_GROUP("async", test_wc_CryptoCb_AsyncPollTlsBothDirections) + +#endif /* WOLFCRYPT_TEST_ASYNC_H */ diff --git a/wolfcrypt/src/async.c b/wolfcrypt/src/async.c index 2c2a66cf866..b049c28514e 100644 --- a/wolfcrypt/src/async.c +++ b/wolfcrypt/src/async.c @@ -27,6 +27,10 @@ #include #include +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) +#include +#endif static WC_ASYNC_DEV* wolfAsync_GetDev(WOLF_EVENT* event) @@ -417,11 +421,51 @@ int wolfAsync_DevCtxInit(WC_ASYNC_DEV* asyncDev, word32 marker, void* heap, /* always clear async device context */ XMEMSET(asyncDev, 0, sizeof(WC_ASYNC_DEV)); +#if defined(WOLF_CRYPTO_CB) && defined(WOLF_CRYPTO_CB_ASYNC_POLL) + asyncDev->cryptocb.devId = INVALID_DEVID; +#endif + /* negative device Id's are invalid */ if (devId >= 0) { asyncDev->marker = marker; asyncDev->heap = heap; +#if defined(WOLF_CRYPTO_CB) && defined(WOLF_CRYPTO_CB_ASYNC_POLL) + /* Route the two async record ciphers (AES, 3DES) to poll completion; + * every other op keeps the re-invoke model. This split follows how + * the TLS record layer resumes each op: + * + * - Record ciphers: Encrypt()/Decrypt() advance encrypt.state to + * CIPHER_STATE_END *before* the WC_PENDING_E check, so on resume the + * op does NOT call EncryptDo again -- it reads the output buffer as + * already filled. Re-invocation would never re-run the cipher, so + * the only way to finish the job is for the poll to fill that buffer + * (wc_CryptoCb_Poll during wolfSSL_AsyncPoll). This is the QAT/Nitrox + * completion model. + * + * - Handshake PK (RSA/ECC/DH/X25519): the TLS_ASYNC_* state machine + * resumes *at* the crypto call and re-invokes it (WC_ASYNC_FLAG_ + * CALL_AGAIN), so the callback is driven to completion by repeated + * calls. Poll routing would send it WC_ALGO_TYPE_ASYNC_POLL, which a + * PK device does not answer -> broken handshake. + * + * devId flows in from the WOLFSSL object: wolfSSL_CTX_SetDevId -> + * SetKeys -> wc_AesInit()/wc_Des3Init() -> here, and is re-stamped on + * every key setup (including rekey/KeyUpdate). + * + * Only route when crypto callbacks are the async backend for bulk + * ciphers. With a hardware/SW backend present that backend completes + * the op, so stamping here would make the poll re-enter a device that + * is not a crypto callback. */ + #if !defined(HAVE_INTEL_QA) && !defined(HAVE_CAVIUM) && \ + !defined(WOLFSSL_ASYNC_CRYPT_SW) + if (marker == WOLFSSL_ASYNC_MARKER_AES || + marker == WOLFSSL_ASYNC_MARKER_3DES) { + asyncDev->cryptocb.devId = devId; + } + #endif +#endif + #ifdef HAVE_CAVIUM ret = NitroxAllocContext(asyncDev, devId, CONTEXT_SSL); #elif defined(HAVE_INTEL_QA) @@ -563,6 +607,17 @@ int wolfAsync_EventPoll(WOLF_EVENT* event, WOLF_EVENT_FLAG flags) event->ret = wolfAsync_DoSw(asyncDev); #endif + #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) + /* Re-enter the crypto callback to complete the job and fill the output + * buffer, like the hardware polls above. Poll only while the event is + * still pending so a completed result is not overwritten. */ + if (asyncDev->cryptocb.devId != INVALID_DEVID && + event->ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { + event->ret = wc_CryptoCb_Poll(asyncDev->cryptocb.devId); + } + #endif + /* If not pending then mark as done */ if (event->ret != WC_NO_ERR_TRACE(WC_PENDING_E)) { event->state = WOLF_EVENT_STATE_DONE; @@ -720,11 +775,27 @@ int wolfAsync_EventQueuePoll(WOLF_EVENT_QUEUE* queue, void* context_filter, event->ret = wolfAsync_DoSw(asyncDev); } #elif defined(WOLF_CRYPTO_CB) || defined(HAVE_PK_CALLBACKS) - /* Crypto/PK callbacks manage their own retry state. - * Leave event->ret as WC_PENDING_E so that - * wolfSSL_AsyncPop can detect the pending state and - * remove the event, allowing the operation to be - * retried with a fresh callback invocation. */ + #if defined(WOLFSSL_ASYNC_CRYPT) && \ + defined(WOLF_CRYPTO_CB) && \ + defined(WOLF_CRYPTO_CB_ASYNC_POLL) + if (asyncDev->cryptocb.devId != INVALID_DEVID) { + /* Re-enter the crypto callback to complete the job and + * fill the output buffer. Poll only while still pending + * so a completed result is not overwritten. */ + if (event->ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { + event->ret = + wc_CryptoCb_Poll(asyncDev->cryptocb.devId); + } + } + else + #endif + { + /* Re-invoke model: crypto/PK callbacks manage their own + * retry state. Leave event->ret as WC_PENDING_E so that + * wolfSSL_AsyncPop can detect the pending state and + * remove the event, allowing the operation to be + * retried with a fresh callback invocation. */ + } #else #warning No async crypt device defined! diff --git a/wolfcrypt/src/cryptocb.c b/wolfcrypt/src/cryptocb.c index be5eccb0139..5d8578c2a6d 100644 --- a/wolfcrypt/src/cryptocb.c +++ b/wolfcrypt/src/cryptocb.c @@ -73,6 +73,14 @@ Crypto Callback Build Options: #include +#if defined(WOLFSSL_ASYNC_CRYPT) && !defined(WOLF_CRYPTO_CB_ASYNC_POLL) && \ + !defined(WOLFSSL_ASYNC_CRYPT_SW) && !defined(HAVE_INTEL_QA) && \ + !defined(HAVE_CAVIUM) && !defined(WOLF_CRYPTO_CB_ASYNC_NO_WARN) + #warning "crypto callbacks with async crypt may not work for TLS. Define \ +WOLF_CRYPTO_CB_ASYNC_POLL to enable it, or WOLF_CRYPTO_CB_ASYNC_NO_WARN to \ +silence." +#endif + #ifdef HAVE_ARIA #include #endif @@ -452,6 +460,35 @@ int wc_CryptoCb_GetDevIdAtIndex(int startIdx) return devId; } +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB_ASYNC_POLL) +/* Returns WC_PENDING_E while in-flight, 0 or an error when done. */ +int wc_CryptoCb_Poll(int devId) +{ + /* Default hard-fails: a missing or unregistered device cannot complete + * the pending job, so never report "no pending" and leave the output + * buffer unfilled. */ + int ret = WC_NO_ERR_TRACE(WC_HW_E); + /* Resolve with WC_ALGO_TYPE_CIPHER, the algo the op was submitted under, so + * a WOLF_CRYPTO_CB_FIND remap lands on the same device as the submit. */ + CryptoCb* dev = wc_CryptoCb_FindDevice(devId, WC_ALGO_TYPE_CIPHER); + if (dev != NULL && dev->cb != NULL) { + wc_CryptoInfo info; + XMEMSET(&info, 0, sizeof(info)); + info.algo_type = WC_ALGO_TYPE_ASYNC_POLL; + ret = dev->cb(devId, &info, dev->ctx); + if (ret == WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE) || + ret == WC_NO_ERR_TRACE(NOT_COMPILED_IN) || + ret == WC_NO_ERR_TRACE(WC_NO_PENDING_E)) { + /* Device cannot complete the in-flight job (no poll support, or it + * reports nothing pending for an op we are polling): fail hard + * rather than let the async layer treat it as done. */ + ret = WC_NO_ERR_TRACE(WC_HW_E); + } + } + return ret; +} +#endif /* WOLFSSL_ASYNC_CRYPT && WOLF_CRYPTO_CB_ASYNC_POLL */ + #ifdef WOLF_CRYPTO_CB_FIND /* Used to register a find device function. Useful for cases where the diff --git a/wolfssl/wolfcrypt/async.h b/wolfssl/wolfcrypt/async.h index e1ac06d9f0d..f3cae1d64e0 100644 --- a/wolfssl/wolfcrypt/async.h +++ b/wolfssl/wolfcrypt/async.h @@ -391,6 +391,12 @@ typedef struct WC_ASYNC_DEV { #elif defined(WOLFSSL_ASYNC_CRYPT_SW) WC_ASYNC_SW sw; #endif +#if defined(WOLF_CRYPTO_CB) && defined(WOLF_CRYPTO_CB_ASYNC_POLL) + /* Crypto callback to re-enter at poll time. INVALID_DEVID: none. */ + struct { + int devId; + } cryptocb; +#endif } WC_ASYNC_DEV; diff --git a/wolfssl/wolfcrypt/cryptocb.h b/wolfssl/wolfcrypt/cryptocb.h index 8d2e587e542..75abc9df809 100644 --- a/wolfssl/wolfcrypt/cryptocb.h +++ b/wolfssl/wolfcrypt/cryptocb.h @@ -759,6 +759,13 @@ typedef int (*CryptoDevCallbackFind)(int devId, int algoType); WOLFSSL_API void wc_CryptoCb_SetDeviceFindCb(CryptoDevCallbackFind cb); #endif +#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WOLF_CRYPTO_CB_ASYNC_POLL) +/* Poll completion for bulk cipher ops: on WC_PENDING_E the callback saves + * the output pointers; wolfSSL_AsyncPoll() re-enters it with + * WC_ALGO_TYPE_ASYNC_POLL to finish the job and fill the buffer. */ +WOLFSSL_LOCAL int wc_CryptoCb_Poll(int devId); +#endif + #ifdef DEBUG_CRYPTOCB WOLFSSL_API void wc_CryptoCb_InfoString(wc_CryptoInfo* info); #endif diff --git a/wolfssl/wolfcrypt/settings.h b/wolfssl/wolfcrypt/settings.h index da911e6508d..472ae2aca37 100644 --- a/wolfssl/wolfcrypt/settings.h +++ b/wolfssl/wolfcrypt/settings.h @@ -3956,6 +3956,11 @@ #error No async backend defined with WOLFSSL_ASYNC_CRYPT! #endif + #if defined(WOLF_CRYPTO_CB_ASYNC_POLL) && defined(WC_ASYNC_NO_CRYPT) + /* Poll routing needs the bulk cipher async marker. */ + #error WOLF_CRYPTO_CB_ASYNC_POLL requires bulk cipher async support + #endif + /* Make sure wolf events are enabled */ #undef HAVE_WOLF_EVENT #define HAVE_WOLF_EVENT diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index 010ebcb7e69..c5c94e70fb2 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -1461,7 +1461,10 @@ enum wc_AlgoType { WC_ALGO_TYPE_SETKEY = 12, WC_ALGO_TYPE_EXPORT_KEY = 13, WC_ALGO_TYPE_SHE = 14, - WC_ALGO_TYPE_MAX = WC_ALGO_TYPE_SHE + /* async: re-enter a crypto callback device to poll a pending operation so + * it can complete the work and fill the output buffer (QAT-style). */ + WC_ALGO_TYPE_ASYNC_POLL = 15, + WC_ALGO_TYPE_MAX = WC_ALGO_TYPE_ASYNC_POLL }; /* KDF types */