crypto: handle XOF output allocation failure

Return an operation error when XOF output allocation fails.

Signed-off-by: Filip Skokan <panva.ip@gmail.com>
PR-URL: #64851
Backport-PR-URL: #65087
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Luigi Pinca <luigipinca@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

Author: panva

src/crypto/crypto_turboshake.cc

@@ -468,7 +468,10 @@ bool TurboShakeTraits::DeriveBits(Environment* env,
                                   ByteSource* out,
                                   CryptoJobMode mode) {
   CHECK_GT(params.output_length, 0);
-  char* buf = MallocOpenSSL<char>(params.output_length);
+  char* buf = static_cast<char*>(OPENSSL_malloc(params.output_length));
+  if (buf == nullptr) {
+    return false;
+  }
 
   const uint8_t* input = reinterpret_cast<const uint8_t*>(params.data.data());
   size_t input_len = params.data.size();
@@ -592,7 +595,10 @@ bool KangarooTwelveTraits::DeriveBits(Environment* env,
     return false;
   }
 
-  char* buf = MallocOpenSSL<char>(params.output_length);
+  char* buf = static_cast<char*>(OPENSSL_malloc(params.output_length));
+  if (buf == nullptr) {
+    return false;
+  }
 
   switch (params.variant) {
     case KangarooTwelveVariant::KT128: