Create benchmarking for KEM API - Kyber512 (#185)

.github/workflows/ci.yml

@@ -158,7 +158,7 @@ jobs:
         target: [ native, aarch64-unknown-linux-gnu, i686-unknown-linux-gnu ]
         args:
           - --all-targets
-          - --release --all-targets --features ring-benchmarks
+          - --release --all-targets
           - --no-default-features --features non-fips
           - --no-default-features --features non-fips,ring-io
           - --no-default-features --features non-fips,ring-sig-verify
@@ -198,6 +198,11 @@ jobs:
         if: ${{ matrix.target != 'native' }}
         run: cross test ${{ matrix.args }} ${{ matrix.target != 'native' && format('--target {0}', matrix.target) || '' }}
 
+      - name: Run extra tests
+        working-directory: ./aws-lc-rs-testing
+        if: ${{ matrix.target == 'native' }}
+        run: cargo test --all-targets
+
   fips-test:
     name: aws-lc-rs fips-tests
     runs-on: ${{ matrix.os }}
@@ -241,7 +246,7 @@ jobs:
         os: [ ubuntu-latest, macOS-latest ]
         args:
           - --no-default-features --features aws-lc-sys,bindgen
-          - --release --all-targets --features ring-benchmarks,bindgen
+          - --release --all-targets --features bindgen
     steps:
       - uses: actions/checkout@v3
         with:
@@ -266,7 +271,7 @@ jobs:
         args:
           - --all-targets
           - --all-targets --features bindgen
-          - --release --all-targets --features ring-benchmarks
+          - --release --all-targets
           - --no-default-features --features non-fips
           - --no-default-features --features non-fips,ring-io
           - --no-default-features --features non-fips,ring-sig-verify
@@ -312,7 +317,7 @@ jobs:
       # coverage immediately.
       - name: Run coverage
         working-directory: ./aws-lc-rs
-        run: cargo llvm-cov --no-fail-fast --fail-under-lines 95 --ignore-filename-regex "aws-lc-sys/*"
+        run: cargo llvm-cov --workspace --no-fail-fast --fail-under-lines 95 --ignore-filename-regex "aws-lc-(fips-)?sys/.*"
 
   mirai-analysis:
     runs-on: ubuntu-latest

Cargo.toml

@@ -2,7 +2,8 @@
 members = [
     "aws-lc-rs",
     "aws-lc-sys",
-    "aws-lc-fips-sys"
+    "aws-lc-fips-sys",
+    "aws-lc-rs-testing"
 ]
 resolver = "2"
 

Makefile

@@ -42,4 +42,7 @@ api-diff-pub:
 clippy:
 	cargo +nightly clippy --all-targets --features bindgen -- -W clippy::all  -W clippy::pedantic
 
-.PHONY: init-aws-lc-sys init-aws-lc-fips-sys init-submodules init update-submodules lic audit format api-diff-main api-diff-pub clippy
+udep:
+	cargo +nightly udeps --all-targets
+
+.PHONY: init-aws-lc-sys init-aws-lc-fips-sys init-submodules init update-submodules lic audit format api-diff-main api-diff-pub clippy udep

aws-lc-rs-testing/Cargo.toml

@@ -0,0 +1,71 @@
+[package]
+name = "aws-lc-rs-testing"
+authors = ["AWS-LibCrypto"]
+version = "0.1.0"
+edition = "2021"
+rust-version = "1.60"
+publish = false
+
+[features]
+ring-benchmarks = []
+openssl-benchmarks = []
+bindgen = ["aws-lc-rs/bindgen"]
+fips = ["aws-lc-rs/fips"]
+asan = ["aws-lc-rs/asan"]
+
+[dependencies]
+aws-lc-rs = { version = "1.0", path = "../aws-lc-rs" }
+
+[dev-dependencies]
+paste = "1.0.13"
+criterion = { version = "0.5.0", features = ["csv_output"] }
+ring = "0.16"
+openssl = { version = "0.10.52", features = ["vendored"] }
+
+[[bench]]
+name = "aead_benchmark"
+harness = false
+
+[[bench]]
+name = "digest_benchmark"
+harness = false
+
+[[bench]]
+name = "hkdf_benchmark"
+harness = false
+
+[[bench]]
+name = "hmac_benchmark"
+harness = false
+
+[[bench]]
+name = "pbkdf2_benchmark"
+harness = false
+
+[[bench]]
+name = "quic_benchmark"
+harness = false
+
+[[bench]]
+name = "rsa_benchmark"
+harness = false
+
+[[bench]]
+name = "ecdsa_benchmark"
+harness = false
+
+[[bench]]
+name = "ed25519_benchmark"
+harness = false
+
+[[bench]]
+name = "agreement_benchmark"
+harness = false
+
+[[bench]]
+name = "cipher_benchmark"
+harness = false
+
+[[bench]]
+name = "kem_benchmark"
+harness = false

aws-lc-rs-testing/Makefile

@@ -0,0 +1,25 @@
+include ../Makefile
+
+UNAME_S := $(shell uname -s)
+
+asan:
+# TODO: This build target produces linker error on Mac.
+# Run specific tests:
+#	RUST_BACKTRACE=1 ASAN_OPTIONS=detect_leaks=1 RUSTFLAGS=-Zsanitizer=address RUSTDOCFLAGS=-Zsanitizer=address cargo +nightly test --test ecdsa_tests              --target `rustc -vV | sed -n 's|host: ||p'`  --features asan
+	RUST_BACKTRACE=1 ASAN_OPTIONS=detect_leaks=1 RUSTFLAGS=-Zsanitizer=address RUSTDOCFLAGS=-Zsanitizer=address cargo +nightly test --lib --bins --tests --examples --target `rustc -vV | sed -n 's|host: ||p'`  --features asan
+
+asan-release:
+# TODO: This build target produces linker error on Mac.
+# Run specific tests:
+#	RUST_BACKTRACE=1 ASAN_OPTIONS=detect_leaks=1 RUSTFLAGS=-Zsanitizer=address RUSTDOCFLAGS=-Zsanitizer=address cargo +nightly test --release --test basic_rsa_test           --target `rustc -vV | sed -n 's|host: ||p'`  --features asan
+	RUST_BACKTRACE=1 ASAN_OPTIONS=detect_leaks=1 RUSTFLAGS=-Zsanitizer=address RUSTDOCFLAGS=-Zsanitizer=address cargo +nightly test --release --lib --bins --tests --examples --target `rustc -vV | sed -n 's|host: ||p'`  --features asan
+
+asan-fips:
+# TODO: This build target produces linker error on Mac.
+# Run specific tests:
+#	RUST_BACKTRACE=1 ASAN_OPTIONS=detect_leaks=1 RUSTFLAGS=-Zsanitizer=address RUSTDOCFLAGS=-Zsanitizer=address cargo +nightly test --test ecdsa_tests          --target `rustc -vV | sed -n 's|host: ||p'` --no-default-features --features fips,asan
+	RUST_BACKTRACE=1 ASAN_OPTIONS=detect_leaks=1 RUSTFLAGS=-Zsanitizer=address RUSTDOCFLAGS=-Zsanitizer=address cargo +nightly test --lib --bins --tests --examples --target `rustc -vV | sed -n 's|host: ||p'` --no-default-features --features fips,asan
+
+ci: format clippy test coverage api-diff-pub
+
+.PHONY: asan asan-fips asan-release ci clippy coverage test

aws-lc-rs-testing/benches/data/kem_benchmarks.txt

@@ -0,0 +1,4 @@
+algorithm = KYBER512_R3
+pk = 115ACE0E64677CBB7DCFC93C16D3A305F67615A488D711AA56698C5663AB7AC9CE66D547C0595F98A43F4650BBE08C364D976789117D34F6AE51AC063CB55C6CA32558227DFEF807D19C30DE414424097F6AA236A1053B4A07A76BE372A5C6B6002791EBE0AFDAF54E1CA237FF545BA68343E745C04AD1639DBC590346B6B9569B56DBBFE53151913066E5C85527DC9468110A136A411497C227DCB8C9B25570B7A0E42AADA6709F23208F5D496EBAB7843F6483BF0C0C73A40296EC2C6440001394C99CA173D5C775B7F415D02A5A26A07407918587C41169F2B7178755ACC27FC8B19C4C4B3FCD41053F2C74C8A10A8321241B2802432875AE808B9EF1365C7B8A52902F1317BA2FB0269F47930672107B4726FEF64547394D3320C8F120B3C2F4725B0305FAB88CC7981FCB09A76A1CBF7F179F43BB0A4C8B0590857F1E69708466C7F8607391E7BC5268BFD3D7A1DFFCB4ECA2A1C9B597593013D5FC4202EC2B74E57AB76BBCF3632BBAF97CDC418A6F16392838CA9BF45DDF023777B7561833C105190F94F302C59B531900BBC816361FAA5B3380CA3A893104CA7388B185671B3E5FE3790E9A626EC46D9B0B33C7A419AF7B32B6859894F575D82AC5456B5490A7AF8FE61046360589ECBA7244236F4123116B6174AA179249A49195B356C72FC6641F0251812EAA98570B046699070E0819DC2713F469137DFC6A3D7B92B298995EE780369153AC366B06D7249CD09E1B3378FB04399CECB8650581D637C79AE67D6F2CAF6ABACF598159A7792CB3C971D1499D2373AD20F63F03BB59ED137384AC61A7155143B8CA4932612EC915E4CA346A9BCE5DD60417C6B2A89B1CC435643F875BDC5A7E5B3481CF919EA09172FEBC46D4FC3FB0CB9591704EE2DBB61844B2F3314A06BB6C6D34005E485CE667BDC7D098586928D2D91340F00419EA401351A240A0B041058BEFB0C2FD32645B7A2DF8F5CBFD873327C978D7B351A28088438837024C52B9C295CD713646FB5D6C0CCFB470734AC2B2BC8123C2C13DF6938E92455A862639FEB8A64B85163E32707E037B38D8AC3922B45187BB65EAFD465FC64A0C5F8F3F9003489415899D59A543D8208C54A3166529B53922
+sk = 6C892B0297A9C7641493F87DAF3533EED61F07F4652066337ED74046DCC71BA03F30960103161F7DEB53A71B11617263FE2A809769CE6D70A85FE600ECE29D7F36A16D331B8B2A9E1DB8C090742DF0739FF060CEB4ECC5AB1C5E55AC97BB66A7F895105D57782B229538E3421544A3421408DBF44910934CC423774F1676FF1C306F97555F57B4AED7A6BAB950A8163C8D318DEA62751BD6ABC5069C06C88F330026A19806A03B97A7696B56DA21827BB4E8DC031152B41B892A9E99ADF6E1963E96578828154F467033846920FBB4B80544E7E8A81AE963CF368C9BA037A8C2AD62E32B6E61C91D75CE005AB30F8099A1F29D7B6305B4DC06E25680BB00992F717FE6C115A8084231CC79DD700EA6912AC7FA0D937BB6A756662230470C189B5AA1653DEB937D5A9C25A21D93B19074FC239D8153539797C7D4AB62649D76AA553736A949022C22C52BAEEC605B32CE9E5B9384903558CA9D6A3ABA90423EEDA01C94198B192A8BA9063497A0C5013307DDD863526471A4D99523EB417F291AAC0C3A581B6DA00732E5E81B1F7C879B1693C13B6F9F7931622429E542AF4069222F045544E0CC4FB24D4448CF2C6596F5CB08624B1185013B6B020892F96BDFD4ADA9179DE727B8D9426E0996B5D34948CE02D0C369B37CBB54D3479ED8B582E9E728929B4C71C9BE11D45B20C4BDC3C74313223F58274E8BA5244447C495950B84CB0C3C273640108A3397944573279328996CDC0C913C958AD620BA8B5E5ECBBB7E13CB9C70BD5AB30EB7488C97001C20498F1D7CC06DA76BF520C658CCADFA2956424557ABEA8AB89239C17833DC3A49B36A9AE9A486940540EB444F97152357E02035939D75A3C025F41A40082382A0733C39B0622B740E407592C62ECAEB1432C445B3703A86F6981A278157EA95A6E92D55E4B972F936C2F0A658280EA2B07A48992DF8937E0A2AC1DCC974FE00AAE1F561FA258E2D259C3E861DCE236039127606FC1CE009003A7BAC942101DCB822B1F3C12BF73238F546E01C36B5A6936192995CC69C63237409CB53C2E35D74890D18885376FA5503B107A2A392115ACE0E64677CBB7DCFC93C16D3A305F67615A488D711AA56698C5663AB7AC9CE66D547C0595F98A43F4650BBE08C364D976789117D34F6AE51AC063CB55C6CA32558227DFEF807D19C30DE414424097F6AA236A1053B4A07A76BE372A5C6B6002791EBE0AFDAF54E1CA237FF545BA68343E745C04AD1639DBC590346B6B9569B56DBBFE53151913066E5C85527DC9468110A136A411497C227DCB8C9B25570B7A0E42AADA6709F23208F5D496EBAB7843F6483BF0C0C73A40296EC2C6440001394C99CA173D5C775B7F415D02A5A26A07407918587C41169F2B7178755ACC27FC8B19C4C4B3FCD41053F2C74C8A10A8321241B2802432875AE808B9EF1365C7B8A52902F1317BA2FB0269F47930672107B4726FEF64547394D3320C8F120B3C2F4725B0305FAB88CC7981FCB09A76A1CBF7F179F43BB0A4C8B0590857F1E69708466C7F8607391E7BC5268BFD3D7A1DFFCB4ECA2A1C9B597593013D5FC4202EC2B74E57AB76BBCF3632BBAF97CDC418A6F16392838CA9BF45DDF023777B7561833C105190F94F302C59B531900BBC816361FAA5B3380CA3A893104CA7388B185671B3E5FE3790E9A626EC46D9B0B33C7A419AF7B32B6859894F575D82AC5456B5490A7AF8FE61046360589ECBA7244236F4123116B6174AA179249A49195B356C72FC6641F0251812EAA98570B046699070E0819DC2713F469137DFC6A3D7B92B298995EE780369153AC366B06D7249CD09E1B3378FB04399CECB8650581D637C79AE67D6F2CAF6ABACF598159A7792CB3C971D1499D2373AD20F63F03BB59ED137384AC61A7155143B8CA4932612EC915E4CA346A9BCE5DD60417C6B2A89B1CC435643F875BDC5A7E5B3481CF919EA09172FEBC46D4FC3FB0CB9591704EE2DBB61844B2F3314A06BB6C6D34005E485CE667BDC7D098586928D2D91340F00419EA401351A240A0B041058BEFB0C2FD32645B7A2DF8F5CBFD873327C978D7B351A28088438837024C52B9C295CD713646FB5D6C0CCFB470734AC2B2BC8123C2C13DF6938E92455A862639FEB8A64B85163E32707E037B38D8AC3922B45187BB65EAFD465FC64A0C5F8F3F9003489415899D59A543D8208C54A3166529B539227FFAD1BC8AF73B7E874956B81C2A2EF0BFABE8DC93D77B2FBC9E0C64EFA01E848626ED79D451140800E03B59B956F8210E556067407D13DC90FA9E8B872BFB8F
+ct = EDF24145E43B4F6DC6BF8332F54E02CAB02DBF3B5605DDC90A15C886AD3ED489462699E4ABED44350BC3757E2696FBFB2534412E8DD201F1E4540A3970B055FE3B0BEC3A71F9E115B3F9F39102065B1CCA8314DCC795E3C0E8FA98EE83CA6628457028A4D09E839E554862CF0B7BF56C5C0A829E8657947945FE9C22564FBAEBC1B3AF350D7955508A26D8A8EB547B8B1A2CF03CCA1AABCE6C3497783B6465BA0B6E7ACBA821195124AEF09E628382A1F914043BE7096E952CBC4FB4AFED13609046117C011FD741EE286C83771690F0AEB50DA0D71285A179B215C6036DEB780F4D16769F72DE16FDADAC73BEFA5BEF8943197F44C59589DC9F4973DE1450BA1D0C3290D6B1D683F294E759C954ABE8A7DA5B1054FD6D21329B8E73D3756AFDA0DCB1FC8B1582D1F90CF275A102ABC6AC699DF0C5870E50A1F989E4E6241B60AAA2ECF9E8E33E0FFCF40FE831E8FDC2E83B52CA7AB6D93F146D29DCA53C7DA1DB4AC4F2DB39EA120D90FA60F4D437C6D00EF483BC94A3175CDA163FC1C2828BE4DBD6430507B584BB5177E171B8DDA9A4293C3200295C803A865D6D2166F66BA5401FB7A0E853168600A2948437E036E3BF19E12FD3F2A2B8B343F784248E8D685EB0AFDE6315338730E7A1001C27D8D2A76FA69D157BA1AC7AD56DA5A8C70FE4B5B8D786DC6FC0566BA8E1B8816334D32A3FB1CE7D4D5E4C332AF7B003D091741A3D5C965292255DFF8ED2BBF1F9116BE50C17B8E548748AD4B2E957BBD1953482A2E1718CEC66CD2C81F572D552B7187885E6B8943D6431413C59EBB7E036048490BE5289E95B20A89E8B159F61A9A9886E147568F4C9021F362F02688A1C8C3BB0D24086880E55B6EDB43F3745D2C166DC1CB743C76FE6BE523A893CC764D16435C37851252A81E2FFBA0F18971A3DEE37D4877CB928E36E5235037A6B2057897D518A5F0E348E3AB6D5B52DFC60757F3B41A4FEC7828F1DEEAF4587CCC8EADF647F4D203B2FAA05A649B582340CB4CACE57A30711BE752FACF0227D0A80C4128442DDC544BE805B9CFE8FE9B1237C80F96787CD9281CCF270C1AFC0670D

aws-lc-rs-testing/benches/kem_benchmark.rs

@@ -0,0 +1,137 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0 OR ISC
+
+use aws_lc_rs::{test, test_file};
+use criterion::{criterion_group, criterion_main, Criterion};
+
+#[allow(non_camel_case_types)]
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+enum Algorithm {
+    KYBER512_R3,
+}
+
+impl From<&str> for Algorithm {
+    fn from(value: &str) -> Self {
+        match value {
+            "KYBER512_R3" => Algorithm::KYBER512_R3,
+            _ => panic!("Unrecognized algorithm: '{value}'"),
+        }
+    }
+}
+
+pub struct KemConfig {
+    algorithm: Algorithm,
+    public_key: Box<[u8]>,
+    secret_key: Box<[u8]>,
+    ciphertext: Box<[u8]>,
+}
+
+impl KemConfig {
+    fn new(algorithm: &str, public_key: &[u8], secret_key: &[u8], ciphertext: &[u8]) -> Self {
+        KemConfig {
+            algorithm: Algorithm::from(algorithm),
+            public_key: Vec::from(public_key).into(),
+            secret_key: Vec::from(secret_key).into(),
+            ciphertext: Vec::from(ciphertext).into(),
+        }
+    }
+}
+
+mod aws_lc_rs_benchmarks {
+
+    use aws_lc_rs::kem;
+
+    use crate::{Algorithm, KemConfig};
+    use kem::{KemAlgorithm, KemPrivateKey, KemPublicKey, KYBER512_R3};
+
+    fn algorithm(config: &KemConfig) -> &'static KemAlgorithm {
+        match config.algorithm {
+            Algorithm::KYBER512_R3 => &KYBER512_R3,
+        }
+    }
+
+    pub fn new_private_key(config: &KemConfig) -> KemPrivateKey {
+        KemPrivateKey::new(algorithm(config), &config.secret_key).unwrap()
+    }
+
+    pub fn new_public_key(config: &KemConfig) -> KemPublicKey {
+        KemPublicKey::new(algorithm(config), &config.public_key).unwrap()
+    }
+
+    pub fn keygen(config: &KemConfig) {
+        let private_key = KemPrivateKey::generate(algorithm(config)).unwrap();
+        let _public_key = private_key.compute_public_key().unwrap();
+    }
+
+    pub fn encapsulate(public_key: &KemPublicKey) {
+        public_key
+            .encapsulate((), |ct, ss| Ok((Vec::from(ct), Vec::from(ss))))
+            .unwrap();
+    }
+
+    pub fn decapsulate(config: &KemConfig, secret_key: &KemPrivateKey) {
+        let mut ciphertext = config.ciphertext.clone();
+        secret_key
+            .decapsulate(&mut ciphertext, (), |ss| Ok(Vec::from(ss)))
+            .unwrap();
+    }
+}
+
+fn bench_kem_keygen(c: &mut Criterion, config: &KemConfig) {
+    let bench_group_name = format!("KEM-{:?}-keygen", config.algorithm);
+
+    let mut group = c.benchmark_group(bench_group_name);
+
+    group.bench_function("AWS-LC", |b| {
+        b.iter(|| {
+            aws_lc_rs_benchmarks::keygen(config);
+        });
+    });
+}
+
+fn bench_kem_encapsulate(c: &mut Criterion, config: &KemConfig) {
+    let bench_group_name = format!("KEM-{:?}-encapsulate", config.algorithm);
+
+    let mut group = c.benchmark_group(bench_group_name);
+
+    group.bench_function("AWS-LC", |b| {
+        b.iter(|| {
+            let public_key = aws_lc_rs_benchmarks::new_public_key(config);
+            aws_lc_rs_benchmarks::encapsulate(&public_key);
+        });
+    });
+}
+
+fn bench_kem_decapsulate(c: &mut Criterion, config: &KemConfig) {
+    let bench_group_name = format!("KEM-{:?}-decapsulate", config.algorithm);
+
+    let mut group = c.benchmark_group(bench_group_name);
+
+    group.bench_function("AWS-LC", |b| {
+        b.iter(|| {
+            let private_key = aws_lc_rs_benchmarks::new_private_key(config);
+            aws_lc_rs_benchmarks::decapsulate(config, &private_key);
+        });
+    });
+}
+
+fn bench_kem(c: &mut Criterion) {
+    test::run(
+        test_file!("data/kem_benchmarks.txt"),
+        |_section, test_case| {
+            let config = KemConfig::new(
+                test_case.consume_string("algorithm").as_str(),
+                test_case.consume_bytes("pk").as_slice(),
+                test_case.consume_bytes("sk").as_slice(),
+                test_case.consume_bytes("ct").as_slice(),
+            );
+            bench_kem_keygen(c, &config);
+            bench_kem_encapsulate(c, &config);
+            bench_kem_decapsulate(c, &config);
+            Ok(())
+        },
+    );
+}
+
+criterion_group!(benches, bench_kem);
+criterion_main!(benches);

aws-lc-rs/benches/rsa_benchmark.rs → aws-lc-rs-testing/benches/rsa_benchmark.rs

@@ -256,5 +256,5 @@ criterion_main!(benches);
 
 #[cfg(not(feature = "ring-sig-verify"))]
 fn main() {
-    println!("This bench requires feature: ring-sig-verify")
+    println!("This bench requires feature: ring-sig-verify");
 }