Intel QuickAssist: multi-device utilization + software-fallback / Cavium fixes

Makefile.am

@@ -25,6 +25,18 @@ ignore_files =
 SUBDIRS_OPT =
 DIST_SUBDIRS_OPT =
 
+# Serialize the build when Intel QuickAssist is enabled. Concurrent QAT user
+# processes (the parallel test binaries that `make -j check` launches) exhaust
+# the device's crypto instances and usdm contiguous memory, so the test phase
+# must run serially. With the non-recursive build this disables -j for the
+# whole invocation, which also matches the QAT driver's build guidance.
+if BUILD_INTEL_QA
+.NOTPARALLEL:
+endif
+if BUILD_INTEL_QA_SYNC
+.NOTPARALLEL:
+endif
+
 # allow supplementary or override flags to be passed at make time:
 AM_CPPFLAGS += $(EXTRA_CPPFLAGS)
 AM_CFLAGS += $(EXTRA_CFLAGS)

src/tls.c

@@ -11020,11 +11020,10 @@ int TLSX_KeyShare_HandlePqcHybridKeyServer(WOLFSSL* ssl,
 
 #ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == 0) {
-        /* Check if the provided kse already contains ECC data and the
-        * last error was WC_PENDING_E. In this case, we already tried to
-        * process ECC kse data. Hence, we have to restore it. */
-        if (keyShareEntry->key != NULL && keyShareEntry->keyLen > 0 &&
-            keyShareEntry->lastRet == WC_NO_ERR_TRACE(WC_PENDING_E)) {
+        /* Restore ECC state from a prior suspended pass. Not gated on
+         * lastRet == WC_PENDING_E: the async layer clears lastRet to 0 on
+         * completion, which would skip the restore and regenerate the key. */
+        if (keyShareEntry->key != NULL && keyShareEntry->keyLen > 0) {
             ecc_kse->key = keyShareEntry->key;
             ecc_kse->keyLen = keyShareEntry->keyLen;
             ecc_kse->pubKey = keyShareEntry->pubKey;
@@ -11143,6 +11142,7 @@ int TLSX_KeyShare_HandlePqcHybridKeyServer(WOLFSSL* ssl,
         else if (ret == WC_NO_ERR_TRACE(WC_PENDING_E)) {
             keyShareEntry->lastRet = WC_PENDING_E;
             keyShareEntry->key = ecc_kse->key;
+            keyShareEntry->keyLen = ecc_kse->keyLen;
             keyShareEntry->pubKey = ecc_kse->pubKey;
             keyShareEntry->pubKeyLen = ecc_kse->pubKeyLen;
             ecc_kse->key = NULL;
@@ -11178,6 +11178,12 @@ int TLSX_KeyShare_HandlePqcHybridKeyServer(WOLFSSL* ssl,
         ssl->arrays->preMasterSz += ssSzPqc;
         keyShareEntry->ke = NULL;
         keyShareEntry->keLen = 0;
+    #ifdef WOLFSSL_ASYNC_CRYPT
+        /* Hybrid encapsulation is fully complete here. Clear the pending
+         * state so the TLS_ASYNC_VERIFY re-drive is skipped and does not
+         * re-enter this handler with the now-freed ke. */
+        keyShareEntry->lastRet = 0;
+    #endif
 
         /* Concatenate the ECDH public key and the PQC KEM ciphertext. Based on
          * the pqc_first flag, the ECDH public key goes before or after the KEM
@@ -11964,6 +11970,18 @@ int TLSX_KeyShare_Setup(WOLFSSL *ssl, KeyShareEntry* clientKSE)
         if (extension != NULL && extension->resp == 1) {
             serverKSE = (KeyShareEntry*)extension->data;
             if (serverKSE != NULL) {
+#if defined(WOLFSSL_HAVE_MLKEM) && !defined(WOLFSSL_MLKEM_NO_ENCAPSULATE)
+                /* Re-drive server hybrid encapsulation on resume. GenKey
+                 * routes a hybrid group to the client generator, and the
+                 * lastRet == 0 path treats the share as done after only the
+                 * ECDH part completed, dropping the KEM ciphertext. ke holds
+                 * the client share until the handler completes and clears it. */
+                if (serverKSE->ke != NULL &&
+                        WOLFSSL_NAMED_GROUP_IS_PQC_HYBRID(serverKSE->group)) {
+                    return TLSX_KeyShare_HandlePqcHybridKeyServer((WOLFSSL*)ssl,
+                            serverKSE, serverKSE->ke, serverKSE->keLen);
+                }
+#endif
                 /* in async case make sure key generation is finalized */
                 if (serverKSE->lastRet == WC_NO_ERR_TRACE(WC_PENDING_E))
                     return TLSX_KeyShare_GenKey((WOLFSSL*)ssl, serverKSE);

tests/api/test_tls13.c

@@ -5488,6 +5488,68 @@ int test_tls13_pqc_hybrid_malformed_ecdh(void)
     return EXPECT_RESULT();
 }
 
+/* Regression test for the async hybrid PQC server key share. Drives a full
+ * TLS 1.3 P-256 + ML-KEM-768 handshake through the software async simulator
+ * while forcing the server's ECDH keygen to complete synchronously so that
+ * only the ECDH shared-secret derivation suspends. This "B-first" ordering is
+ * what Intel QAT exhibits and previously dropped the server's KEM ciphertext,
+ * failing the handshake with SSL_connect -173. */
+int test_tls13_pqc_hybrid_async_server(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_TLS13) && defined(WOLFSSL_ASYNC_CRYPT) && \
+    defined(WOLFSSL_ASYNC_CRYPT_SW) && \
+    defined(HAVE_MANUAL_MEMIO_TESTS_DEPENDENCIES) && \
+    defined(WOLFSSL_HAVE_MLKEM) && defined(WOLFSSL_PQC_HYBRIDS) && \
+    !defined(WOLFSSL_MLKEM_NO_ENCAPSULATE) && \
+    !defined(WOLFSSL_MLKEM_NO_MAKE_KEY) && \
+    !defined(WOLFSSL_MLKEM_NO_DECAPSULATE) && \
+    !defined(WOLFSSL_NO_ML_KEM_768) && defined(HAVE_ECC) && \
+    !defined(NO_WOLFSSL_CLIENT) && !defined(NO_WOLFSSL_SERVER) && \
+    (!defined(NO_ECC256) || defined(HAVE_ALL_CURVES)) && !defined(NO_ECC_SECP) && \
+    !defined(WC_ECC_NONBLOCK) && !defined(WOLFSSL_SP_NONBLOCK)
+    /* Non-blocking math livelocks the hybrid async re-drive; not applicable. */
+    struct test_memio_ctx test_ctx;
+    WOLFSSL_CTX *ctx_c = NULL, *ctx_s = NULL;
+    WOLFSSL *ssl_c = NULL, *ssl_s = NULL;
+    int group = WOLFSSL_SECP256R1MLKEM768;
+    int devId = INVALID_DEVID;
+
+    /* Open the software async device so the handshake runs the async path. */
+    ExpectIntEQ(wolfAsync_DevOpen(&devId), 0);
+
+    XMEMSET(&test_ctx, 0, sizeof(test_ctx));
+    ExpectIntEQ(test_memio_setup(&test_ctx, &ctx_c, &ctx_s, &ssl_c, &ssl_s,
+                    wolfTLSv1_3_client_method, wolfTLSv1_3_server_method), 0);
+
+    ExpectIntEQ(wolfSSL_SetDevId(ssl_c, devId), WOLFSSL_SUCCESS);
+    ExpectIntEQ(wolfSSL_SetDevId(ssl_s, devId), WOLFSSL_SUCCESS);
+
+    /* Negotiate the P-256 + ML-KEM-768 hybrid group on both ends. */
+    ExpectIntEQ(wolfSSL_set_groups(ssl_c, &group, 1), WOLFSSL_SUCCESS);
+    ExpectIntEQ(wolfSSL_set_groups(ssl_s, &group, 1), WOLFSSL_SUCCESS);
+
+    /* Force the server's ECDH keygen to run synchronously so that only the
+     * ECDH shared-secret derivation suspends (the QAT "B-first" ordering). */
+    wolfAsync_SwForceSyncType(ASYNC_SW_ECC_MAKE);
+
+    ExpectIntEQ(test_memio_do_handshake(ssl_c, ssl_s, 20, NULL), 0);
+
+    /* Server selected and completed the hybrid group end-to-end. */
+    ExpectIntEQ(ssl_s->namedGroup, WOLFSSL_SECP256R1MLKEM768);
+
+    /* Restore default simulator ordering for subsequent tests. */
+    wolfAsync_SwForceSyncType(ASYNC_SW_NONE);
+
+    wolfSSL_free(ssl_c);
+    wolfSSL_CTX_free(ctx_c);
+    wolfSSL_free(ssl_s);
+    wolfSSL_CTX_free(ctx_s);
+    wolfAsync_DevClose(&devId);
+#endif
+    return EXPECT_RESULT();
+}
+
 /* Test that a TLS 1.3 NewSessionTicket with a ticket shorter than ID_LEN
  * (32 bytes) does not cause an unsigned integer underflow / OOB read in
  * SetTicket. Uses a full memio handshake, then injects a crafted

tests/api/test_tls13.h

@@ -85,6 +85,7 @@ int test_tls13_AEAD_limit_KU_aes128_gcm_sha256(void);
 int test_tls13_AEAD_limit_KU_aes256_gcm_sha384(void);
 int test_tls13_AEAD_limit_KU_aes128_ccm_sha256(void);
 int test_tls13_AEAD_limit_KU_aes128_ccm_8_sha256(void);
+int test_tls13_pqc_hybrid_async_server(void);
 
 #define TEST_TLS13_DECLS                                        \
     TEST_DECL_GROUP("tls13", test_tls13_apis),                  \
@@ -147,6 +148,7 @@ int test_tls13_AEAD_limit_KU_aes128_ccm_8_sha256(void);
     TEST_DECL_GROUP("tls13", test_tls13_AEAD_limit_KU_aes128_gcm_sha256), \
     TEST_DECL_GROUP("tls13", test_tls13_AEAD_limit_KU_aes256_gcm_sha384), \
     TEST_DECL_GROUP("tls13", test_tls13_AEAD_limit_KU_aes128_ccm_sha256), \
-    TEST_DECL_GROUP("tls13", test_tls13_AEAD_limit_KU_aes128_ccm_8_sha256)
+    TEST_DECL_GROUP("tls13", test_tls13_AEAD_limit_KU_aes128_ccm_8_sha256), \
+    TEST_DECL_GROUP("tls13", test_tls13_pqc_hybrid_async_server)
 
 #endif /* WOLFCRYPT_TEST_TLS13_H */

tests/utils.c

@@ -186,6 +186,13 @@ int test_memio_do_handshake(WOLFSSL *ssl_c, WOLFSSL *ssl_s,
                 if (err == WC_NO_ERR_TRACE(MP_WOULDBLOCK)) {
                     /* retry non-blocking math */
                 }
+            #ifdef WOLFSSL_ASYNC_CRYPT
+                else if (err == WC_NO_ERR_TRACE(WC_PENDING_E)) {
+                    ret = wolfSSL_AsyncPoll(ssl_c, WOLF_POLL_FLAG_CHECK_HW);
+                    if (ret < 0)
+                        return -1;
+                }
+            #endif
                 else if (err != WOLFSSL_ERROR_WANT_READ &&
                          err != WOLFSSL_ERROR_WANT_WRITE) {
                     char buff[WOLFSSL_MAX_ERROR_SZ];
@@ -207,6 +214,13 @@ int test_memio_do_handshake(WOLFSSL *ssl_c, WOLFSSL *ssl_s,
                 if (err == WC_NO_ERR_TRACE(MP_WOULDBLOCK)) {
                     /* retry non-blocking math */
                 }
+            #ifdef WOLFSSL_ASYNC_CRYPT
+                else if (err == WC_NO_ERR_TRACE(WC_PENDING_E)) {
+                    ret = wolfSSL_AsyncPoll(ssl_s, WOLF_POLL_FLAG_CHECK_HW);
+                    if (ret < 0)
+                        return -1;
+                }
+            #endif
                 else if (err != WOLFSSL_ERROR_WANT_READ &&
                          err != WOLFSSL_ERROR_WANT_WRITE) {
                     char buff[WOLFSSL_MAX_ERROR_SZ];

wolfcrypt/src/async.c

@@ -67,6 +67,15 @@ static WC_ASYNC_DEV* wolfAsync_GetDev(WOLF_EVENT* event)
 /* Allow way to have async SW code included, and disabled at run-time */
 static int wolfAsyncSwDisabled = 0; /* default off */
 
+/* Test hook: an op of this type runs synchronously instead of suspending.
+ * Default ASYNC_SW_NONE means no type is forced (no real op uses NONE). */
+static int wolfAsyncSwForceSyncType = ASYNC_SW_NONE;
+
+WOLFSSL_TEST_VIS void wolfAsync_SwForceSyncType(int type)
+{
+    wolfAsyncSwForceSyncType = type;
+}
+
 
 static int wolfAsync_DoSw(WC_ASYNC_DEV* asyncDev)
 {
@@ -312,6 +321,9 @@ int wc_AsyncSwInit(WC_ASYNC_DEV* dev, int type)
     if (dev) {
         WC_ASYNC_SW* sw = &dev->sw;
         if (sw->type == ASYNC_SW_NONE) {
+            /* Test hook: force this op type to run synchronously. */
+            if (type == wolfAsyncSwForceSyncType)
+                return 0;
             sw->type = type;
             return 1;
         }
@@ -686,6 +698,9 @@ int wolfAsync_EventQueuePoll(WOLF_EVENT_QUEUE* queue, void* context_filter,
                         if (ret != 0) {
                             break;
                         }
+                        /* buffer flushed: restart indexing to avoid writing
+                         * past multi_req.req[CAVIUM_MAX_POLL] */
+                        req_count = 0;
                     }
             #else
                 #if defined(HAVE_INTEL_QA)