crypto: handle DH operation failures

Report DH failures instead of aborting or returning an empty secret.

Signed-off-by: Filip Skokan <panva.ip@gmail.com>
PR-URL: #64851
Backport-PR-URL: #65087
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Luigi Pinca <luigipinca@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

Author: panva

src/crypto/crypto_dh.cc

@@ -332,6 +332,10 @@ void ComputeSecret(const FunctionCallbackInfo<Value>& args) {
   }
 
   auto dp = dh.computeSecret(key);
+  if (!dp) {
+    return THROW_ERR_CRYPTO_OPERATION_FAILED(env,
+                                             "Failed to compute shared secret");
+  }
 
   Local<Value> buffer;
   if (DataPointerToBuffer(env, std::move(dp)).ToLocal(&buffer)) {
@@ -349,8 +353,8 @@ void SetPublicKey(const FunctionCallbackInfo<Value>& args) {
   if (!buf.CheckSizeInt32()) [[unlikely]]
     return THROW_ERR_OUT_OF_RANGE(env, "buf is too big");
   BignumPointer num(buf.data(), buf.size());
-  CHECK(num);
-  CHECK(dh.setPublicKey(std::move(num)));
+  if (!num || !dh.setPublicKey(std::move(num)))
+    return THROW_ERR_INVALID_ARG_VALUE(env, "Invalid public key");
 }
 
 void SetPrivateKey(const FunctionCallbackInfo<Value>& args) {
@@ -363,8 +367,8 @@ void SetPrivateKey(const FunctionCallbackInfo<Value>& args) {
   if (!buf.CheckSizeInt32()) [[unlikely]]
     return THROW_ERR_OUT_OF_RANGE(env, "buf is too big");
   BignumPointer num(buf.data(), buf.size());
-  CHECK(num);
-  CHECK(dh.setPrivateKey(std::move(num)));
+  if (!num || !dh.setPrivateKey(std::move(num)))
+    return THROW_ERR_INVALID_ARG_VALUE(env, "Invalid private key");
 }
 
 void Check(const FunctionCallbackInfo<Value>& args) {