Add internal APIs for ML-DSA (aws#1999)

This PR adds the internal functions from FIPS 204: Module-Lattice-Based
Digital Signature Standard. We base this implementation on the upstream
reference implementation of ML-DSA
pq-crystals/dilithium@444cdcc.
However, the upstream commit only includes implementation of
`ML-DSA.Sign_internal` and `ML-DSA.Verify_internal`, so we also include
`ML-DSA.KeyGen_internal` to complete the implementation.

Changes:
* Implementation of external FIPS 204 ML-DSA algorithm functions
* Addition of internal functions to the `pqdsa` API:
    * ml_dsa_65_keypair_internal
    * ml_dsa_65_sign_internal
    * ml_dsa_65_verify_internal
* Addition of KATs from seeds (KATs from
https://github.com/post-quantum-cryptography/KAT/tree/main)
    * New testing framework to run KATs from seeds (as in ACVP)
* Removal of `pq_custom_randombytes.{c/h}`

crypto/CMakeLists.txt

@@ -479,7 +479,6 @@ add_library(
   rand_extra/forkunsafe.c
   rand_extra/fuchsia.c
   rand_extra/rand_extra.c
-  rand_extra/pq_custom_randombytes.c
   rand_extra/trusty.c
   rand_extra/windows.c
   rc4/rc4.c

crypto/dilithium/ml_dsa.c

@@ -32,6 +32,14 @@ int ml_dsa_65_keypair(uint8_t *public_key   /* OUT */,
   return (crypto_sign_keypair(&params, public_key, private_key) == 0);
 }
 
+int ml_dsa_65_keypair_internal(uint8_t *public_key   /* OUT */,
+                               uint8_t *private_key  /* OUT */,
+                               const uint8_t *seed   /* IN */) {
+  ml_dsa_params params;
+  ml_dsa_65_params_init(&params);
+  return crypto_sign_keypair_internal(&params, public_key, private_key, seed) == 0;
+}
+
 int ml_dsa_65_sign(const uint8_t *private_key /* IN */,
                    uint8_t *sig               /* OUT */,
                    size_t *sig_len            /* OUT */,
@@ -45,6 +53,20 @@ int ml_dsa_65_sign(const uint8_t *private_key /* IN */,
                                ctx_string, ctx_string_len, private_key) == 0;
 }
 
+int ml_dsa_65_sign_internal(const uint8_t *private_key  /* IN */,
+                            uint8_t *sig                /* OUT */,
+                            size_t *sig_len             /* OUT */,
+                            const uint8_t *message      /* IN */,
+                            size_t message_len          /* IN */,
+                            const uint8_t *pre          /* IN */,
+                            size_t pre_len              /* IN */,
+                            uint8_t *rnd                /* IN */) {
+  ml_dsa_params params;
+  ml_dsa_65_params_init(&params);
+  return crypto_sign_signature_internal(&params, sig, sig_len, message, message_len,
+                                        pre, pre_len, rnd, private_key) == 0;
+}
+
 int ml_dsa_65_verify(const uint8_t *public_key /* IN */,
                      const uint8_t *sig        /* IN */,
                      size_t sig_len            /* IN */,
@@ -57,3 +79,16 @@ int ml_dsa_65_verify(const uint8_t *public_key /* IN */,
   return crypto_sign_verify(&params, sig, sig_len, message, message_len,
                             ctx_string, ctx_string_len, public_key) == 0;
 }
+
+int ml_dsa_65_verify_internal(const uint8_t *public_key /* IN */,
+                              const uint8_t *sig        /* IN */,
+                              size_t sig_len            /* IN */,
+                              const uint8_t *message    /* IN */,
+                              size_t message_len        /* IN */,
+                              const uint8_t *pre        /* IN */,
+                              size_t pre_len            /* IN */) {
+  ml_dsa_params params;
+  ml_dsa_65_params_init(&params);
+  return crypto_sign_verify_internal(&params, sig, sig_len, message, message_len,
+                                     pre, pre_len, public_key) == 0;
+}

crypto/dilithium/ml_dsa.h

@@ -15,22 +15,39 @@
 #define MLDSA65_KEYGEN_SEED_BYTES 32
 #define MLDSA65_SIGNATURE_SEED_BYTES 32
 
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
 int ml_dsa_65_keypair(uint8_t *public_key,
-                      uint8_t *private_key);
+                      uint8_t *secret_key);
+
+int ml_dsa_65_keypair_internal(uint8_t *public_key,
+                               uint8_t *private_key,
+                               const uint8_t *seed);
 
 int ml_dsa_65_sign(const uint8_t *private_key,
-                   uint8_t *sig,
-                   size_t *sig_len,
-                   const uint8_t *message,
-                   size_t message_len,
-                   const uint8_t *ctx_string,
-                   size_t ctx_string_len);
+                   uint8_t *sig, size_t *sig_len,
+                   const uint8_t *message, size_t message_len,
+                   const uint8_t *ctx_string, size_t ctx_len);
+
+int ml_dsa_65_sign_internal(const uint8_t *private_key,
+                            uint8_t *sig, size_t *sig_len,
+                            const uint8_t *message, size_t message_len,
+                            const uint8_t *pre, size_t pre_len,
+                            uint8_t *rnd);
 
 int ml_dsa_65_verify(const uint8_t *public_key,
-                     const uint8_t *sig,
-                     size_t sig_len,
-                     const uint8_t *message,
-                     size_t message_len,
-                     const uint8_t *ctx_string,
-                     size_t ctx_string_len);
+                     const uint8_t *sig, size_t sig_len,
+                     const uint8_t *message, size_t message_len,
+                     const uint8_t *ctx_string, size_t ctx_string_len);
+
+int ml_dsa_65_verify_internal(const uint8_t *public_key,
+                              const uint8_t *sig, size_t sig_len,
+                              const uint8_t *message, size_t message_len,
+                              const uint8_t *pre, size_t pre_len);
+#if defined(__cplusplus)
+}
+#endif
+
 #endif

crypto/dilithium/p_pqdsa_test.cc

@@ -10,7 +10,6 @@
 #include "../test/test_util.h"
 
 #include <vector>
-#include "../crypto/evp_extra/internal.h"
 #include "../fipsmodule/evp/internal.h"
 #include "../internal.h"
 #include "internal.h"
@@ -19,7 +18,6 @@
 
 #include "../test/file_test.h"
 #include "../test/test_util.h"
-#include "../rand_extra/pq_custom_randombytes.h"
 #include "ml_dsa.h"
 
 // mldsa65kPublicKey is an example ML-DSA-65 public key
@@ -357,7 +355,7 @@ CMP_VEC_AND_PTR(vec, pkey->pkey.pqdsa_key->public_key, len)
 CMP_VEC_AND_PTR(vec, pkey->pkey.pqdsa_key->private_key, len)
 
 static const struct PQDSATestVector parameterSet[] = {
-  {"MLDSA65", NID_MLDSA65, 1952, 4032, 3309,  "dilithium/kat/mldsa65.txt", mldsa65kPublicKey, mldsa65kPublicKeySPKI, 1974},
+  {"MLDSA65", NID_MLDSA65, 1952, 4032, 3309,  "dilithium/kat/MLDSA_65_hedged_pure.txt", mldsa65kPublicKey, mldsa65kPublicKeySPKI, 1974},
 };
 
 class PQDSAParameterTest : public testing::TestWithParam<PQDSATestVector> {};
@@ -366,72 +364,74 @@ INSTANTIATE_TEST_SUITE_P(All, PQDSAParameterTest, testing::ValuesIn(parameterSet
                          [](const testing::TestParamInfo<PQDSATestVector> &params)
                              -> std::string { return params.param.name; });
 
-
 TEST_P(PQDSAParameterTest, KAT) {
   std::string kat_filepath = "crypto/";
   kat_filepath += GetParam().kat_filename;
 
   FileTestGTest(kat_filepath.c_str(), [&](FileTest *t) {
     std::string count, mlen, smlen;
-    std::vector<uint8_t> seed, msg, pk, sk, sm;
+    std::vector<uint8_t> xi, rng, seed, msg, pk, sk, sm, ctxstr;
 
     ASSERT_TRUE(t->GetAttribute(&count, "count"));
+    ASSERT_TRUE(t->GetBytes(&xi, "xi"));
+    ASSERT_TRUE(t->GetBytes(&rng, "rng"));
     ASSERT_TRUE(t->GetBytes(&seed, "seed"));
-    ASSERT_TRUE(t->GetAttribute(&mlen, "mlen"));
-    ASSERT_TRUE(t->GetBytes(&msg, "msg"));
     ASSERT_TRUE(t->GetBytes(&pk, "pk"));
     ASSERT_TRUE(t->GetBytes(&sk, "sk"));
-    ASSERT_TRUE(t->GetAttribute(&smlen, "smlen"));
+    ASSERT_TRUE(t->GetBytes(&msg, "msg"));
+    ASSERT_TRUE(t->GetAttribute(&mlen, "mlen"));
     ASSERT_TRUE(t->GetBytes(&sm, "sm"));
+    ASSERT_TRUE(t->GetAttribute(&smlen, "smlen"));
+    ASSERT_TRUE(t->GetBytes(&ctxstr, "ctx"));
 
     size_t pk_len = GetParam().public_key_len;
     size_t sk_len = GetParam().private_key_len;
     size_t sig_len = GetParam().signature_len;
 
-    // The KAT files generated by the dilithium team use the optional APIs that
-    // create a signature for a message m and append the message to the end of
-    // the signature. We only want to bring the APIs that create and verify just
-    // the signature, therefore each signature is a constant
-    // DILITHIUM3_SIGNATURE_BYTES and we truncate the KAT's signed message down,
-    // "sm" down to a constant DILITHIUM3_SIGNATURE_BYTES.
-
+    std::vector<uint8_t> pub(pk_len);
+    std::vector<uint8_t> priv(sk_len);
     std::vector<uint8_t> signature(sig_len);
-    sm.resize(sig_len);
 
-    // Convert string read from KAT to int
+    std::string name = GetParam().name;
     size_t mlen_int = std::stoi(mlen);
 
-    // Here we fix the DRBG (AES-CTR) so that we are able to seed it with the
-    // seed from the KAT (testing only)
-    pq_custom_randombytes_use_deterministic_for_testing();
-    pq_custom_randombytes_init_for_testing(seed.data());
-
-    // Generate our dilithium public and private key pair
-    bssl::UniquePtr<EVP_PKEY_CTX> pctx(EVP_PKEY_CTX_new_id(EVP_PKEY_PQDSA, nullptr));
-    ASSERT_TRUE(pctx);
-    ASSERT_TRUE(EVP_PKEY_CTX_pqdsa_set_params(pctx.get(),GetParam().nid));
-    ASSERT_TRUE(EVP_PKEY_keygen_init(pctx.get()));
-    EVP_PKEY *raw = nullptr;
-    ASSERT_TRUE(EVP_PKEY_keygen(pctx.get(), &raw));
-    bssl::UniquePtr<EVP_PKEY> pkey(raw);
-
-    // Compare the expected public/secret key from KATs with generated values
-    CMP_VEC_AND_PKEY_PUBLIC(pk, pkey, pk_len);
-    CMP_VEC_AND_PKEY_SECRET(sk, pkey, sk_len);
-
-    // Generate a signature for the message
-    // We use EVP_DigestSign because dilithium supports the use of
-    // non-hash-then-sign (just like ed25519) so we first init EVP_DigestSign
-    // WITHOUT a hash function.
-    bssl::ScopedEVP_MD_CTX ctx;
-    ASSERT_TRUE(EVP_DigestSignInit(ctx.get(), nullptr, nullptr, nullptr, pkey.get()));
-    ASSERT_TRUE(EVP_DigestSign(ctx.get(), signature.data(), &sig_len, msg.data(), mlen_int));
-    EXPECT_EQ(Bytes(sm), Bytes(signature.data(), sig_len));
-    ctx.Reset();
-
-    // Verify the signature
-    ASSERT_TRUE(EVP_DigestVerifyInit(ctx.get(), nullptr, nullptr, nullptr, pkey.get()));
-    ASSERT_TRUE(EVP_DigestVerify(ctx.get(), signature.data(), sig_len, msg.data(), mlen_int));
+    // The KATs provide the signed message, which is the signature appended with
+    // the message that was signed. We use the ML-DSA APIs for sign_signature
+    // and not sign_message, which return the signature without the appended
+    // message, so we truncate the signed messaged to |sig_len|.
+    sm.resize(sig_len);
+
+    // Generate key pair from seed xi and assert that public and private keys
+    // are equal to expected values from KAT
+    if (name == "MLDSA65") {
+      ASSERT_TRUE(ml_dsa_65_keypair_internal(pub.data(), priv.data(), xi.data()));
+    }
+    EXPECT_EQ(Bytes(pub), Bytes(pk));
+    EXPECT_EQ(Bytes(priv), Bytes(sk));
+
+    // Prepare m_prime = (0 || ctxlen || ctx)
+    // See both FIPS 204: Algorithm 2 line 10 and FIPS 205: Algorithm 22 line 8
+    uint8_t m_prime[257];
+    size_t m_prime_len = ctxstr.size() + 2;
+    m_prime[0] = 0;
+    m_prime[1] = ctxstr.size();
+    ASSERT_TRUE(ctxstr.size() <= 255);
+    OPENSSL_memcpy(m_prime + 2 , ctxstr.data(), ctxstr.size());
+
+    // Generate signature by signing |msg|, assert that signature is equal
+    // to expected value from KAT, then verify signature.
+    if (name == "MLDSA65") {
+      ASSERT_TRUE(ml_dsa_65_sign_internal(priv.data(),
+                                          signature.data(), &sig_len,
+                                          msg.data(), mlen_int,
+                                          m_prime, m_prime_len,
+                                          rng.data()));
+      ASSERT_EQ(Bytes(signature), Bytes(sm));
+      ASSERT_TRUE(ml_dsa_65_verify_internal(pub.data(),
+                                            signature.data(), sig_len,
+                                            msg.data(), mlen_int,
+                                            m_prime, m_prime_len));
+    }
   });
 }
 

crypto/dilithium/pqcrystals_dilithium_ref_common/README.md

@@ -4,19 +4,18 @@ The source code in this folder implements ML-DSA as defined in FIPS 204 Module-L
 
 **Source code origin and modifications** 
 
-The source code was imported from a branch of the official repository of the Crystals-Dilithium team: https://github.com/pq-crystals/dilithium. The code was taken at [commit](https://github.com/pq-crystals/dilithium/commit/cbcd8753a43402885c90343cd6335fb54712cda1) as of 10/01/2024. At the moment, only the reference C implementation is imported.
+The source code was imported from a branch of the official repository of the Crystals-Dilithium team: https://github.com/pq-crystals/dilithium. The code was taken at [commit](https://github.com/pq-crystals/dilithium/commit/444cdcc84eb36b66fe27b3a2529ee48f6d8150c2) as of 10/29/2024. At the moment, only the reference C implementation is imported.
 
 The code was refactored in [this PR](https://github.com/aws/aws-lc/pull/1910) by parameterizing all functions that depend on values that are specific to a parameter set, i.e., that directly or indirectly depend on the value of `DILITHIUM_MODE`. To do this, in `params.h` we defined a structure that holds those ML-DSA parameters and functions
 that initialize a given structure with values corresponding to a parameter set. This structure is then passed to every function that requires it as a function argument. In addition, the following changes were made to the source code in `pqcrystals_dilithium_ref_common` directory:
 
 - `randombytes.{h|c}` are deleted because we are using the randomness generation functions provided by AWS-LC.
-- `sign.c`: calls to `randombytes` function is replaced with calls to `pq_custom_randombytes` and the appropriate header file is included (`crypto/rand_extra/pq_custom_randombytes.h`).
+- `sign.c`: calls to `randombytes` function is replaced with calls to `RAND_bytes` and the appropriate header file is included (`openssl/rand.h`).
 - `ntt.c`, `poly.c`, `reduce.c`, `reduce.h`: have been modified with a code refactor. The function `fqmul` has been added to bring mode code consistency with Kyber/ML-KEM. See https://github.com/aws/aws-lc/pull/1748 for more details on this change.
 - `reduce.c`: a small fix to documentation has been made on the bounds of `reduce32`.
 - `poly.c`: a small fix to documentation has been made on the bounds of `poly_reduce`.
 - `polyvec.c`: a small fix to documentation has been made on the bounds of `polyveck_reduce`.
 
 **Testing** 
 
-The KATs were obtained from https://github.com/pq-crystals/dilithium/tree/master/ref/nistkat.
-To compile the KAT programs on Linux or macOS, go to the `ref/` directory and run `make nistkat`. This will produce executables within `nistkat` which once executed will produce the KATs: `PQCsignKAT_Dilithium2.rsp`, `PQCsignKAT_Dilithium3.rsp`,`PQCsignKAT_Dilithium5.rsp`.
+The KATs were obtained from https://github.com/post-quantum-cryptography/KAT. We select the KATs for the signing mode `hedged`, which derives the signing private random seed (rho) pseudorandomly from the signer's private key, the message to be signed, and a 256-bit string `rnd` which is generated at random. The `pure` variant of these KATs were used, as they provide test vector inputs for "pure" i.e., non-pre-hashed messages. The KAT files have been modified to insert linebreaks between each test vector set.

crypto/dilithium/pqcrystals_dilithium_ref_common/params.h

@@ -1,8 +1,6 @@
 #ifndef PARAMS_H
 #define PARAMS_H
 
-#define DILITHIUM_RANDOMIZED_SIGNING
-
 // The only defined parameters are those that don't depend
 // on the parameter set. All other parameters are specified
 // in ml_dsa_params structure that is unique for each parameter
@@ -20,8 +18,8 @@
 
 // Structure for ML-DSA parameters that depend on the parameter set.
 typedef struct {
-  size_t k;
-  size_t l;
+  uint8_t k;
+  uint8_t l;
   size_t eta;
   size_t tau;
   size_t beta;